2023-09-22T12:00:18.134 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-22T12:00:18.134 DEBUG:teuthology.run:Teuthology command: teuthology --description rgw/verify/{centos_latest clusters/fixed-2 frontend/civetweb ignore-pg-availability msgr-failures/few objectstore/bluestore-bitmap overrides proto/https rgw_pool_type/ec-profile sharding$/{default} striping$/{stripe-greater-than-chunk} tasks/{0-install bucket-check cls ragweed reshard s3tests-java s3tests versioning} validater/valgrind} --owner scheduled_cfsnyder@teuthology --name cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi --verbose --archive /home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400937 -- /home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400937/orig.config.yaml 2023-09-22T12:00:18.181 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-22T12:00:18.281 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400937 branch: wip-cfsnyder-bucket-check-pacific description: rgw/verify/{centos_latest clusters/fixed-2 frontend/civetweb ignore-pg-availability msgr-failures/few objectstore/bluestore-bitmap overrides proto/https rgw_pool_type/ec-profile sharding$/{default} striping$/{stripe-greater-than-chunk} tasks/{0-install bucket-check cls ragweed reshard s3tests-java s3tests versioning} validater/valgrind} email: null first_in_suite: false job_id: '7400937' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: wip-cfsnyder-bucket-check-pacific ceph: conf: client: debug rgw: 20 rgw crypt require ssl: false rgw crypt s3 kms backend: testing rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= rgw lc debug interval: 10 rgw max chunk size: 4194304 rgw obj stripe size: 6291456 setgroup: ceph setuser: ceph global: mon client directed command retry: 5 ms inject socket failures: 5000 osd heartbeat grace: 40 osd_max_pg_log_entries: 10 osd_min_pg_log_entries: 10 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon osd crush smoke test: false osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd fast shutdown: false osd objectstore: bluestore osd shutdown pgref assert: true flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(PG_AVAILABILITY\) - \(POOL_APP_NOT_ENABLED\) - \(OSD_SLOW_PING_TIME sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 openssl_keys: rgw.client.0: ca: root client: client.0 embed-key: true rgw.client.1: ca: root client: client.1 embed-key: true root: client: client.0 cn: teuthology install: - client.0 - client.1 key-type: rsa:4096 rgw: client.0: ssl certificate: rgw.client.0 client.1: ssl certificate: rgw.client.1 compression type: random ec-data-pool: true erasure_code_profile: crush-failure-domain: osd k: 3 m: 1 name: testprofile frontend: civetweb storage classes: LUKEWARM, FROZEN s3tests: slow_backend: true selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-cfsnyder-bucket-check-pacific sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 owner: scheduled_cfsnyder@teuthology priority: 101 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - node-exporter.b sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 sleep_before_teardown: 0 suite: rgw suite_branch: wip-cfsnyder-bucket-check-pacific suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 targets: smithi069.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLr7fwikwEzZnPMBsRWaZzWO3JCYp457eHBmJUE5Yu1Yo7J6l/XDwLs2hjtKMDese0SvCViCpFdttPpGjGBWMbg= smithi174.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNYkpIEXE/phKHjSx0EOl4K5VhwOmLgwIRe17NNcy4gHY1AOMrfwEgRVydlL1JyjV1OzIekR1QObSJeoBChy/2U= tasks: - install: null - ceph: null - openssl_keys: null - rgw: client.0: null - workunit: clients: client.0: - rgw/run-bucket-check.sh - workunit: clients: client.0: - cls/test_cls_lock.sh - cls/test_cls_log.sh - cls/test_cls_refcount.sh - cls/test_cls_rgw.sh - cls/test_cls_cmpomap.sh - cls/test_cls_2pc_queue.sh - rgw/test_rgw_gc_log.sh - rgw/test_rgw_obj.sh - rgw/test_rgw_throttle.sh - rgw/test_librgw_file.sh - ragweed: client.0: default-branch: ceph-pacific rgw_server: client.0 stages: prepare,check - workunit: clients: client.0: - rgw/run-reshard.sh - s3tests-java: client.0: force-branch: ceph-pacific force-repo: https://github.com/ceph/java_s3tests.git - s3tests: client.0: force-branch: ceph-pacific rgw_server: client.0 - workunit: clients: client.0: - rgw/run-versioning.sh teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-22_11:28:34 tube: smithi user: cfsnyder verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-22T12:00:18.282 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa; will attempt to use it 2023-09-22T12:00:18.282 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa/tasks 2023-09-22T12:00:18.283 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-22T12:00:18.284 INFO:teuthology.task.internal:Checking packages... 2023-09-22T12:00:18.307 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '31c126b7c1c0c06b66d9d19c5bfc51c46b503392' 2023-09-22T12:00:18.308 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-22T12:00:18.308 INFO:teuthology.packaging:ref: None 2023-09-22T12:00:18.308 INFO:teuthology.packaging:tag: None 2023-09-22T12:00:18.308 INFO:teuthology.packaging:branch: wip-cfsnyder-bucket-check-pacific 2023-09-22T12:00:18.308 INFO:teuthology.packaging:sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T12:00:18.308 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-cfsnyder-bucket-check-pacific 2023-09-22T12:00:18.561 INFO:teuthology.task.internal:Found packages for ceph version 16.2.14-5.g31c126b7 2023-09-22T12:00:18.561 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-22T12:00:18.568 INFO:teuthology.task.internal:no buildpackages task found 2023-09-22T12:00:18.568 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-22T12:00:18.574 INFO:teuthology.task.internal:Saving configuration 2023-09-22T12:00:18.591 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-22T12:00:18.598 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-22T12:00:18.622 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi069.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400937', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-22 11:55:25.840182', 'locked_by': 'scheduled_cfsnyder@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLr7fwikwEzZnPMBsRWaZzWO3JCYp457eHBmJUE5Yu1Yo7J6l/XDwLs2hjtKMDese0SvCViCpFdttPpGjGBWMbg='} 2023-09-22T12:00:18.648 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi174.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400937', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-22 11:55:25.841923', 'locked_by': 'scheduled_cfsnyder@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNYkpIEXE/phKHjSx0EOl4K5VhwOmLgwIRe17NNcy4gHY1AOMrfwEgRVydlL1JyjV1OzIekR1QObSJeoBChy/2U='} 2023-09-22T12:00:18.648 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-22T12:00:18.654 INFO:teuthology.task.internal:roles: ubuntu@smithi069.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a'] 2023-09-22T12:00:18.654 INFO:teuthology.task.internal:roles: ubuntu@smithi174.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'node-exporter.b'] 2023-09-22T12:00:18.654 INFO:teuthology.run_tasks:Running task console_log... 2023-09-22T12:00:18.726 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f14f7ad2ee0>, signals=[15]) 2023-09-22T12:00:18.727 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-22T12:00:18.733 INFO:teuthology.task.internal:Opening connections... 2023-09-22T12:00:18.733 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi069.front.sepia.ceph.com 2023-09-22T12:00:18.734 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:00:18.806 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi174.front.sepia.ceph.com 2023-09-22T12:00:18.807 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:00:18.877 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-22T12:00:18.884 DEBUG:teuthology.orchestra.run.smithi069:> uname -m 2023-09-22T12:00:18.907 INFO:teuthology.orchestra.run.smithi069.stdout:x86_64 2023-09-22T12:00:18.907 DEBUG:teuthology.orchestra.run.smithi069:> cat /etc/os-release 2023-09-22T12:00:18.967 INFO:teuthology.orchestra.run.smithi069.stdout:NAME="CentOS Stream" 2023-09-22T12:00:18.967 INFO:teuthology.orchestra.run.smithi069.stdout:VERSION="8" 2023-09-22T12:00:18.968 INFO:teuthology.orchestra.run.smithi069.stdout:ID="centos" 2023-09-22T12:00:18.968 INFO:teuthology.orchestra.run.smithi069.stdout:ID_LIKE="rhel fedora" 2023-09-22T12:00:18.968 INFO:teuthology.orchestra.run.smithi069.stdout:VERSION_ID="8" 2023-09-22T12:00:18.968 INFO:teuthology.orchestra.run.smithi069.stdout:PLATFORM_ID="platform:el8" 2023-09-22T12:00:18.968 INFO:teuthology.orchestra.run.smithi069.stdout:PRETTY_NAME="CentOS Stream 8" 2023-09-22T12:00:18.968 INFO:teuthology.orchestra.run.smithi069.stdout:ANSI_COLOR="0;31" 2023-09-22T12:00:18.968 INFO:teuthology.orchestra.run.smithi069.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-09-22T12:00:18.968 INFO:teuthology.orchestra.run.smithi069.stdout:HOME_URL="https://centos.org/" 2023-09-22T12:00:18.968 INFO:teuthology.orchestra.run.smithi069.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-22T12:00:18.968 INFO:teuthology.orchestra.run.smithi069.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-22T12:00:18.968 INFO:teuthology.orchestra.run.smithi069.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-09-22T12:00:18.969 INFO:teuthology.lock.ops:Updating smithi069.front.sepia.ceph.com on lock server 2023-09-22T12:00:18.987 DEBUG:teuthology.orchestra.run.smithi174:> uname -m 2023-09-22T12:00:19.009 INFO:teuthology.orchestra.run.smithi174.stdout:x86_64 2023-09-22T12:00:19.009 DEBUG:teuthology.orchestra.run.smithi174:> cat /etc/os-release 2023-09-22T12:00:19.071 INFO:teuthology.orchestra.run.smithi174.stdout:NAME="CentOS Stream" 2023-09-22T12:00:19.071 INFO:teuthology.orchestra.run.smithi174.stdout:VERSION="8" 2023-09-22T12:00:19.071 INFO:teuthology.orchestra.run.smithi174.stdout:ID="centos" 2023-09-22T12:00:19.072 INFO:teuthology.orchestra.run.smithi174.stdout:ID_LIKE="rhel fedora" 2023-09-22T12:00:19.072 INFO:teuthology.orchestra.run.smithi174.stdout:VERSION_ID="8" 2023-09-22T12:00:19.072 INFO:teuthology.orchestra.run.smithi174.stdout:PLATFORM_ID="platform:el8" 2023-09-22T12:00:19.072 INFO:teuthology.orchestra.run.smithi174.stdout:PRETTY_NAME="CentOS Stream 8" 2023-09-22T12:00:19.072 INFO:teuthology.orchestra.run.smithi174.stdout:ANSI_COLOR="0;31" 2023-09-22T12:00:19.072 INFO:teuthology.orchestra.run.smithi174.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-09-22T12:00:19.073 INFO:teuthology.orchestra.run.smithi174.stdout:HOME_URL="https://centos.org/" 2023-09-22T12:00:19.073 INFO:teuthology.orchestra.run.smithi174.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-22T12:00:19.073 INFO:teuthology.orchestra.run.smithi174.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-22T12:00:19.073 INFO:teuthology.orchestra.run.smithi174.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-09-22T12:00:19.073 INFO:teuthology.lock.ops:Updating smithi174.front.sepia.ceph.com on lock server 2023-09-22T12:00:19.103 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-22T12:00:19.114 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-22T12:00:19.121 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-22T12:00:19.121 DEBUG:teuthology.orchestra.run.smithi069:> test '!' -e /home/ubuntu/cephtest 2023-09-22T12:00:19.123 DEBUG:teuthology.orchestra.run.smithi174:> test '!' -e /home/ubuntu/cephtest 2023-09-22T12:00:19.144 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-22T12:00:19.150 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-22T12:00:19.150 DEBUG:teuthology.orchestra.run.smithi069:> test -z $(ls -A /var/lib/ceph) 2023-09-22T12:00:19.185 DEBUG:teuthology.orchestra.run.smithi174:> test -z $(ls -A /var/lib/ceph) 2023-09-22T12:00:19.217 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-22T12:00:19.265 INFO:teuthology.run_tasks:Running task kernel... 2023-09-22T12:00:19.280 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-22T12:00:19.281 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-22T12:00:19.281 DEBUG:teuthology.orchestra.run.smithi069:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-22T12:00:19.282 DEBUG:teuthology.orchestra.run.smithi174:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-22T12:00:19.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T12:00:19.303 DEBUG:teuthology.orchestra.run.smithi069:> uname -r 2023-09-22T12:00:19.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T12:00:19.304 DEBUG:teuthology.orchestra.run.smithi174:> uname -r 2023-09-22T12:00:19.364 INFO:teuthology.orchestra.run.smithi069.stdout:4.18.0-489.el8.x86_64 2023-09-22T12:00:19.364 INFO:teuthology.task.kernel:Running kernel on smithi069: 4.18.0-489.el8.x86_64 2023-09-22T12:00:19.365 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum install -y kernel 2023-09-22T12:00:19.367 INFO:teuthology.orchestra.run.smithi174.stdout:4.18.0-489.el8.x86_64 2023-09-22T12:00:19.368 INFO:teuthology.task.kernel:Running kernel on smithi174: 4.18.0-489.el8.x86_64 2023-09-22T12:00:19.368 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum install -y kernel 2023-09-22T12:17:35.575 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 8 - AppStream 33 kB/s | 33 MB 17:13 2023-09-22T12:17:44.033 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 8 - BaseOS 20 MB/s | 47 MB 00:02 2023-09-22T12:17:51.074 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 8 - Extras 59 kB/s | 18 kB 00:00 2023-09-22T12:17:51.458 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 8 - Extras common packages 33 kB/s | 6.9 kB 00:00 2023-09-22T12:17:51.768 INFO:teuthology.orchestra.run.smithi174.stdout:Copr repo for python3-asyncssh owned by ceph 22 kB/s | 3.5 kB 00:00 2023-09-22T12:17:53.079 INFO:teuthology.orchestra.run.smithi174.stdout:Extra Packages for Enterprise Linux 14 MB/s | 16 MB 00:01 2023-09-22T12:17:56.969 INFO:teuthology.orchestra.run.smithi174.stdout:lab-extras 477 kB/s | 24 kB 00:00 2023-09-22T12:18:00.501 INFO:teuthology.orchestra.run.smithi174.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-09-22T12:18:00.501 INFO:teuthology.orchestra.run.smithi174.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-09-22T12:18:00.752 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T12:18:00.753 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T12:18:00.753 INFO:teuthology.orchestra.run.smithi174.stdout: Package Architecture Version Repository Size 2023-09-22T12:18:00.753 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T12:18:00.753 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T12:18:00.753 INFO:teuthology.orchestra.run.smithi174.stdout: kernel x86_64 4.18.0-513.el8 baseos 10 M 2023-09-22T12:18:00.754 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2023-09-22T12:18:00.754 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-core x86_64 4.18.0-513.el8 baseos 43 M 2023-09-22T12:18:00.754 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules x86_64 4.18.0-513.el8 baseos 35 M 2023-09-22T12:18:00.754 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T12:18:00.754 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T12:18:00.754 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T12:18:00.754 INFO:teuthology.orchestra.run.smithi174.stdout:Install 3 Packages 2023-09-22T12:18:00.754 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T12:18:00.755 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 88 M 2023-09-22T12:18:00.755 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 96 M 2023-09-22T12:18:00.755 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T12:18:01.886 INFO:teuthology.orchestra.run.smithi174.stdout:(1/3): kernel-4.18.0-513.el8.x86_64.rpm 9.2 MB/s | 10 MB 00:01 2023-09-22T12:18:02.970 INFO:teuthology.orchestra.run.smithi174.stdout:(2/3): kernel-modules-4.18.0-513.el8.x86_64.rpm 16 MB/s | 35 MB 00:02 2023-09-22T12:18:03.510 INFO:teuthology.orchestra.run.smithi174.stdout:(3/3): kernel-core-4.18.0-513.el8.x86_64.rpm 16 MB/s | 43 MB 00:02 2023-09-22T12:18:03.510 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:18:03.510 INFO:teuthology.orchestra.run.smithi174.stdout:Total 32 MB/s | 88 MB 00:02 2023-09-22T12:18:04.186 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T12:18:04.606 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T12:18:04.606 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T12:18:05.831 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T12:18:05.858 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T12:18:08.628 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T12:18:09.078 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:18:12.706 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:18:13.094 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:18:16.474 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:18:16.784 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:18:38.529 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:18:38.552 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:18:38.999 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:18:38.999 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:18:38.999 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-core-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:18:39.841 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:18:39.841 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T12:18:39.841 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T12:18:39.841 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-4.18.0-513.el8.x86_64 kernel-core-4.18.0-513.el8.x86_64 2023-09-22T12:18:39.841 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules-4.18.0-513.el8.x86_64 2023-09-22T12:18:39.842 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T12:18:39.842 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T12:18:39.990 DEBUG:teuthology.orchestra.run.smithi174:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T12:18:40.247 INFO:teuthology.orchestra.run.smithi174.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T12:18:40.248 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T12:18:40.248 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-513.el8.x86_64 2023-09-22T12:18:40.248 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-09-22T12:18:40.248 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-09-22T12:18:40.248 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-09-22T12:18:40.248 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi174.front.sepia.ceph.com, path=None, version=distro) 2023-09-22T12:18:40.248 DEBUG:teuthology.orchestra.run.smithi174:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T12:18:40.457 INFO:teuthology.orchestra.run.smithi174.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T12:18:40.457 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T12:18:40.457 DEBUG:teuthology.orchestra.run.smithi174:> sudo rpm -qi grub2-tools 2023-09-22T12:18:40.490 INFO:teuthology.orchestra.run.smithi174.stdout:Name : grub2-tools 2023-09-22T12:18:40.490 INFO:teuthology.orchestra.run.smithi174.stdout:Epoch : 1 2023-09-22T12:18:40.490 INFO:teuthology.orchestra.run.smithi174.stdout:Version : 2.02 2023-09-22T12:18:40.490 INFO:teuthology.orchestra.run.smithi174.stdout:Release : 129.el8 2023-09-22T12:18:40.490 INFO:teuthology.orchestra.run.smithi174.stdout:Architecture: x86_64 2023-09-22T12:18:40.490 INFO:teuthology.orchestra.run.smithi174.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-09-22T12:18:40.490 INFO:teuthology.orchestra.run.smithi174.stdout:Group : System Environment/Base 2023-09-22T12:18:40.490 INFO:teuthology.orchestra.run.smithi174.stdout:Size : 9265586 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:License : GPLv3+ 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:Build Host : kbuilder.bsys.centos.org 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:Relocations : (not relocatable) 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:Packager : CentOS BuildSystem 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:Vendor : CentOS 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:URL : http://www.gnu.org/software/grub/ 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:Summary : Support tools for GRUB. 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:Description : 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T12:18:40.491 INFO:teuthology.orchestra.run.smithi174.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-09-22T12:18:40.492 INFO:teuthology.orchestra.run.smithi174.stdout:customizable bootloader with modular architecture. It supports a rich 2023-09-22T12:18:40.492 INFO:teuthology.orchestra.run.smithi174.stdout:variety of kernel formats, file systems, computer architectures and 2023-09-22T12:18:40.492 INFO:teuthology.orchestra.run.smithi174.stdout:hardware devices. 2023-09-22T12:18:40.492 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T12:18:40.492 INFO:teuthology.orchestra.run.smithi174.stdout:This subpackage provides tools for support of all platforms. 2023-09-22T12:18:40.493 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-09-22T12:18:40.493 INFO:teuthology.task.kernel:Updating grub on smithi174 to boot 4.18.0-513.el8.x86_64 2023-09-22T12:18:40.493 DEBUG:teuthology.orchestra.run.smithi174:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-09-22T12:18:41.948 INFO:teuthology.orchestra.run.smithi174.stderr:Generating grub configuration file ... 2023-09-22T12:18:44.546 INFO:teuthology.orchestra.run.smithi174.stderr:done 2023-09-22T12:18:44.548 DEBUG:teuthology.orchestra.run.smithi174:> mktemp 2023-09-22T12:18:44.578 INFO:teuthology.orchestra.run.smithi174.stdout:/tmp/tmp.xAWULqbQT0 2023-09-22T12:18:44.578 DEBUG:teuthology.orchestra.run.smithi174:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.xAWULqbQT0 2023-09-22T12:18:44.651 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod 0666 /tmp/tmp.xAWULqbQT0 2023-09-22T12:18:44.831 DEBUG:teuthology.orchestra.remote:smithi174:/tmp/tmp.xAWULqbQT0 is 6KB 2023-09-22T12:18:44.880 DEBUG:teuthology.orchestra.run.smithi174:> rm -fr /tmp/tmp.xAWULqbQT0 2023-09-22T12:18:44.898 DEBUG:teuthology.orchestra.run.smithi174:> sudo /bin/ls /boot/loader/entries || true 2023-09-22T12:18:44.963 INFO:teuthology.orchestra.run.smithi174.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-09-22T12:18:44.963 INFO:teuthology.orchestra.run.smithi174.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-09-22T12:18:44.963 INFO:teuthology.orchestra.run.smithi174.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-09-22T12:18:44.963 INFO:teuthology.orchestra.run.smithi174.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64.conf 2023-09-22T12:18:44.964 DEBUG:teuthology.orchestra.run.smithi174:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64 2023-09-22T12:18:45.170 DEBUG:teuthology.orchestra.run.smithi174:> sudo shutdown -r now 2023-09-22T12:19:15.173 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-09-22T12:19:15.173 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2023-09-22T12:19:15.174 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:19:39.718 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - AppStream 29 kB/s | 33 MB 19:17 2023-09-22T12:19:47.627 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - BaseOS 25 MB/s | 47 MB 00:01 2023-09-22T12:19:54.568 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - Extras 83 kB/s | 18 kB 00:00 2023-09-22T12:19:54.952 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - Extras common packages 34 kB/s | 6.9 kB 00:00 2023-09-22T12:19:55.261 INFO:teuthology.orchestra.run.smithi069.stdout:Copr repo for python3-asyncssh owned by ceph 22 kB/s | 3.5 kB 00:00 2023-09-22T12:19:57.363 INFO:teuthology.orchestra.run.smithi069.stdout:Extra Packages for Enterprise Linux 8.1 MB/s | 16 MB 00:01 2023-09-22T12:20:01.162 INFO:teuthology.orchestra.run.smithi069.stdout:lab-extras 429 kB/s | 24 kB 00:00 2023-09-22T12:20:04.695 INFO:teuthology.orchestra.run.smithi069.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-09-22T12:20:04.695 INFO:teuthology.orchestra.run.smithi069.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-09-22T12:20:04.952 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T12:20:04.952 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T12:20:04.952 INFO:teuthology.orchestra.run.smithi069.stdout: Package Architecture Version Repository Size 2023-09-22T12:20:04.952 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T12:20:04.953 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T12:20:04.953 INFO:teuthology.orchestra.run.smithi069.stdout: kernel x86_64 4.18.0-513.el8 baseos 10 M 2023-09-22T12:20:04.953 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2023-09-22T12:20:04.953 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-core x86_64 4.18.0-513.el8 baseos 43 M 2023-09-22T12:20:04.953 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-modules x86_64 4.18.0-513.el8 baseos 35 M 2023-09-22T12:20:04.953 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T12:20:04.953 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T12:20:04.953 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T12:20:04.953 INFO:teuthology.orchestra.run.smithi069.stdout:Install 3 Packages 2023-09-22T12:20:04.954 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T12:20:04.954 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 88 M 2023-09-22T12:20:04.954 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 96 M 2023-09-22T12:20:04.954 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T12:20:06.846 INFO:teuthology.orchestra.run.smithi069.stdout:(1/3): kernel-4.18.0-513.el8.x86_64.rpm 7.5 MB/s | 10 MB 00:01 2023-09-22T12:20:07.622 INFO:teuthology.orchestra.run.smithi069.stdout:(2/3): kernel-core-4.18.0-513.el8.x86_64.rpm 20 MB/s | 43 MB 00:02 2023-09-22T12:20:08.580 INFO:teuthology.orchestra.run.smithi069.stdout:(3/3): kernel-modules-4.18.0-513.el8.x86_64.rpm 12 MB/s | 35 MB 00:03 2023-09-22T12:20:08.581 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:20:08.581 INFO:teuthology.orchestra.run.smithi069.stdout:Total 24 MB/s | 88 MB 00:03 2023-09-22T12:20:09.248 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T12:20:09.664 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T12:20:09.664 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T12:20:10.872 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T12:20:10.896 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T12:20:13.636 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T12:20:14.124 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:20:15.176 DEBUG:teuthology.orchestra.remote:timed out 2023-09-22T12:20:17.713 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:20:18.115 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:20:21.483 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:20:21.786 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:20:24.177 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2023-09-22T12:20:24.178 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:20:24.536 DEBUG:teuthology.orchestra.run.smithi174:> true 2023-09-22T12:20:24.839 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi174.front.sepia.ceph.com' 2023-09-22T12:20:24.839 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-513.el8.x86_64"... 2023-09-22T12:20:24.840 DEBUG:teuthology.orchestra.run.smithi174:> uname -r 2023-09-22T12:20:24.857 INFO:teuthology.orchestra.run.smithi174.stdout:4.18.0-513.el8.x86_64 2023-09-22T12:20:24.857 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-513.el8.x86_64 vs 4.18.0-513.el8.x86_64 2023-09-22T12:20:24.857 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-09-22T12:20:24.857 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-09-22T12:20:25.857 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-22T12:20:25.858 DEBUG:teuthology.orchestra.run.smithi174:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-22T12:20:25.953 INFO:teuthology.orchestra.run.smithi174.stdout:ttyS1 2023-09-22T12:20:25.966 DEBUG:teuthology.parallel:result is None 2023-09-22T12:20:43.586 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:20:43.608 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:20:43.952 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:20:43.952 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:20:43.952 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-core-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:20:45.056 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:20:45.056 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T12:20:45.056 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T12:20:45.056 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-4.18.0-513.el8.x86_64 kernel-core-4.18.0-513.el8.x86_64 2023-09-22T12:20:45.057 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-modules-4.18.0-513.el8.x86_64 2023-09-22T12:20:45.057 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T12:20:45.057 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T12:20:45.225 DEBUG:teuthology.orchestra.run.smithi069:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T12:20:45.436 INFO:teuthology.orchestra.run.smithi069.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T12:20:45.436 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T12:20:45.437 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-513.el8.x86_64 2023-09-22T12:20:45.437 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-09-22T12:20:45.437 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-09-22T12:20:45.437 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-09-22T12:20:45.437 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi069.front.sepia.ceph.com, path=None, version=distro) 2023-09-22T12:20:45.437 DEBUG:teuthology.orchestra.run.smithi069:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T12:20:45.645 INFO:teuthology.orchestra.run.smithi069.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T12:20:45.645 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T12:20:45.645 DEBUG:teuthology.orchestra.run.smithi069:> sudo rpm -qi grub2-tools 2023-09-22T12:20:45.679 INFO:teuthology.orchestra.run.smithi069.stdout:Name : grub2-tools 2023-09-22T12:20:45.679 INFO:teuthology.orchestra.run.smithi069.stdout:Epoch : 1 2023-09-22T12:20:45.679 INFO:teuthology.orchestra.run.smithi069.stdout:Version : 2.02 2023-09-22T12:20:45.679 INFO:teuthology.orchestra.run.smithi069.stdout:Release : 129.el8 2023-09-22T12:20:45.679 INFO:teuthology.orchestra.run.smithi069.stdout:Architecture: x86_64 2023-09-22T12:20:45.679 INFO:teuthology.orchestra.run.smithi069.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:Group : System Environment/Base 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:Size : 9265586 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:License : GPLv3+ 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:Build Host : kbuilder.bsys.centos.org 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:Relocations : (not relocatable) 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:Packager : CentOS BuildSystem 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:Vendor : CentOS 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:URL : http://www.gnu.org/software/grub/ 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:Summary : Support tools for GRUB. 2023-09-22T12:20:45.680 INFO:teuthology.orchestra.run.smithi069.stdout:Description : 2023-09-22T12:20:45.681 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T12:20:45.681 INFO:teuthology.orchestra.run.smithi069.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-09-22T12:20:45.681 INFO:teuthology.orchestra.run.smithi069.stdout:customizable bootloader with modular architecture. It supports a rich 2023-09-22T12:20:45.681 INFO:teuthology.orchestra.run.smithi069.stdout:variety of kernel formats, file systems, computer architectures and 2023-09-22T12:20:45.681 INFO:teuthology.orchestra.run.smithi069.stdout:hardware devices. 2023-09-22T12:20:45.681 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T12:20:45.681 INFO:teuthology.orchestra.run.smithi069.stdout:This subpackage provides tools for support of all platforms. 2023-09-22T12:20:45.682 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-09-22T12:20:45.682 INFO:teuthology.task.kernel:Updating grub on smithi069 to boot 4.18.0-513.el8.x86_64 2023-09-22T12:20:45.682 DEBUG:teuthology.orchestra.run.smithi069:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-09-22T12:20:46.997 INFO:teuthology.orchestra.run.smithi069.stderr:Generating grub configuration file ... 2023-09-22T12:20:49.162 INFO:teuthology.orchestra.run.smithi069.stderr:done 2023-09-22T12:20:49.163 DEBUG:teuthology.orchestra.run.smithi069:> mktemp 2023-09-22T12:20:49.186 INFO:teuthology.orchestra.run.smithi069.stdout:/tmp/tmp.TGHSnFWyqF 2023-09-22T12:20:49.187 DEBUG:teuthology.orchestra.run.smithi069:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.TGHSnFWyqF 2023-09-22T12:20:49.266 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod 0666 /tmp/tmp.TGHSnFWyqF 2023-09-22T12:20:49.422 DEBUG:teuthology.orchestra.remote:smithi069:/tmp/tmp.TGHSnFWyqF is 6KB 2023-09-22T12:20:49.470 DEBUG:teuthology.orchestra.run.smithi069:> rm -fr /tmp/tmp.TGHSnFWyqF 2023-09-22T12:20:49.488 DEBUG:teuthology.orchestra.run.smithi069:> sudo /bin/ls /boot/loader/entries || true 2023-09-22T12:20:49.581 INFO:teuthology.orchestra.run.smithi069.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-09-22T12:20:49.581 INFO:teuthology.orchestra.run.smithi069.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-09-22T12:20:49.581 INFO:teuthology.orchestra.run.smithi069.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-09-22T12:20:49.581 INFO:teuthology.orchestra.run.smithi069.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64.conf 2023-09-22T12:20:49.582 DEBUG:teuthology.orchestra.run.smithi069:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64 2023-09-22T12:20:49.700 DEBUG:teuthology.orchestra.run.smithi069:> sudo shutdown -r now 2023-09-22T12:21:19.703 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-09-22T12:21:19.703 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2023-09-22T12:21:19.704 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:22:19.707 DEBUG:teuthology.orchestra.remote:timed out 2023-09-22T12:22:28.708 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2023-09-22T12:22:28.708 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:22:29.037 DEBUG:teuthology.orchestra.run.smithi069:> true 2023-09-22T12:22:29.357 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi069.front.sepia.ceph.com' 2023-09-22T12:22:29.357 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-513.el8.x86_64"... 2023-09-22T12:22:29.357 DEBUG:teuthology.orchestra.run.smithi069:> uname -r 2023-09-22T12:22:29.374 INFO:teuthology.orchestra.run.smithi069.stdout:4.18.0-513.el8.x86_64 2023-09-22T12:22:29.374 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-513.el8.x86_64 vs 4.18.0-513.el8.x86_64 2023-09-22T12:22:29.374 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-09-22T12:22:29.375 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-09-22T12:22:30.376 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-22T12:22:30.376 DEBUG:teuthology.orchestra.run.smithi069:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-22T12:22:30.469 INFO:teuthology.orchestra.run.smithi069.stdout:ttyS1 2023-09-22T12:22:30.480 DEBUG:teuthology.parallel:result is None 2023-09-22T12:22:30.480 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-22T12:22:30.488 INFO:teuthology.task.internal:Creating test directory... 2023-09-22T12:22:30.488 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-22T12:22:30.490 DEBUG:teuthology.orchestra.run.smithi174:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-22T12:22:30.515 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-22T12:22:30.521 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-22T12:22:30.531 INFO:teuthology.task.internal:Creating archive directory... 2023-09-22T12:22:30.531 DEBUG:teuthology.orchestra.run.smithi069:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-22T12:22:30.549 DEBUG:teuthology.orchestra.run.smithi174:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-22T12:22:30.600 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-22T12:22:30.606 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-22T12:22:30.607 DEBUG:teuthology.orchestra.run.smithi069:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-22T12:22:30.613 DEBUG:teuthology.orchestra.run.smithi174:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-22T12:22:30.643 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:22:30.653 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:22:30.679 INFO:teuthology.orchestra.run.smithi174.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:22:30.690 INFO:teuthology.orchestra.run.smithi174.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:22:30.692 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-22T12:22:30.698 INFO:teuthology.task.internal:Configuring sudo... 2023-09-22T12:22:30.699 DEBUG:teuthology.orchestra.run.smithi069:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-22T12:22:30.701 DEBUG:teuthology.orchestra.run.smithi174:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-22T12:22:30.730 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-22T12:22:30.739 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-22T12:22:30.739 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-22T12:22:30.768 DEBUG:teuthology.orchestra.run.smithi174:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-22T12:22:30.790 DEBUG:teuthology.orchestra.run.smithi069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:22:30.871 DEBUG:teuthology.orchestra.run.smithi069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:22:30.926 DEBUG:teuthology.orchestra.run.smithi069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:22:31.011 DEBUG:teuthology.orchestra.run.smithi069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:22:31.052 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T12:22:31.052 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-22T12:22:31.148 DEBUG:teuthology.orchestra.run.smithi174:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:22:31.199 DEBUG:teuthology.orchestra.run.smithi174:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:22:31.252 DEBUG:teuthology.orchestra.run.smithi174:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:22:31.341 DEBUG:teuthology.orchestra.run.smithi174:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:22:31.385 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T12:22:31.385 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-22T12:22:31.454 DEBUG:teuthology.orchestra.run.smithi069:> sudo service rsyslog restart 2023-09-22T12:22:31.457 DEBUG:teuthology.orchestra.run.smithi174:> sudo service rsyslog restart 2023-09-22T12:22:31.533 INFO:teuthology.orchestra.run.smithi069.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-22T12:22:31.578 INFO:teuthology.orchestra.run.smithi174.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-22T12:22:31.968 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-22T12:22:31.976 INFO:teuthology.task.internal:Starting timer... 2023-09-22T12:22:31.976 INFO:teuthology.run_tasks:Running task pcp... 2023-09-22T12:22:32.025 INFO:teuthology.run_tasks:Running task selinux... 2023-09-22T12:22:32.032 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-22T12:22:32.033 DEBUG:teuthology.orchestra.run.smithi069:> sudo service auditd rotate 2023-09-22T12:22:32.115 INFO:teuthology.orchestra.run.smithi069.stdout:Rotating logs: 2023-09-22T12:22:32.117 DEBUG:teuthology.orchestra.run.smithi174:> sudo service auditd rotate 2023-09-22T12:22:32.188 INFO:teuthology.orchestra.run.smithi174.stdout:Rotating logs: 2023-09-22T12:22:32.190 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-22T12:22:32.190 DEBUG:teuthology.orchestra.run.smithi069:> /usr/sbin/getenforce 2023-09-22T12:22:32.216 INFO:teuthology.orchestra.run.smithi069.stdout:Permissive 2023-09-22T12:22:32.216 DEBUG:teuthology.orchestra.run.smithi174:> /usr/sbin/getenforce 2023-09-22T12:22:32.245 INFO:teuthology.orchestra.run.smithi174.stdout:Permissive 2023-09-22T12:22:32.246 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi069.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi174.front.sepia.ceph.com': 'permissive'} 2023-09-22T12:22:32.246 DEBUG:teuthology.orchestra.run.smithi069:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-22T12:22:32.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T12:22:32.283 DEBUG:teuthology.orchestra.run.smithi174:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-22T12:22:32.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T12:22:32.314 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-22T12:22:32.314 DEBUG:teuthology.orchestra.run.smithi069:> sudo /usr/sbin/setenforce permissive 2023-09-22T12:22:32.350 DEBUG:teuthology.orchestra.run.smithi174:> sudo /usr/sbin/setenforce permissive 2023-09-22T12:22:32.382 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-22T12:22:32.391 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-22T12:22:32.397 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-22T12:22:32.525 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-22T12:22:32.541 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-22T12:22:32.542 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi069.front.sepia.ceph.com,smithi174.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-22T12:42:20.843 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi069.front.sepia.ceph.com'), Remote(name='ubuntu@smithi174.front.sepia.ceph.com')] 2023-09-22T12:42:20.844 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2023-09-22T12:42:20.845 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:42:20.915 DEBUG:teuthology.orchestra.run.smithi069:> true 2023-09-22T12:42:20.985 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi069.front.sepia.ceph.com' 2023-09-22T12:42:20.986 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2023-09-22T12:42:20.986 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:42:21.055 DEBUG:teuthology.orchestra.run.smithi174:> true 2023-09-22T12:42:21.124 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi174.front.sepia.ceph.com' 2023-09-22T12:42:21.125 INFO:teuthology.run_tasks:Running task clock... 2023-09-22T12:42:21.135 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-22T12:42:21.135 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-22T12:42:21.136 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-22T12:42:21.138 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-22T12:42:21.138 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-22T12:42:21.168 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-22T12:42:21.182 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-22T12:42:21.196 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-22T12:42:21.209 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-22T12:42:21.250 INFO:teuthology.orchestra.run.smithi069.stderr:sudo: ntpd: command not found 2023-09-22T12:42:21.261 INFO:teuthology.orchestra.run.smithi069.stdout:506 Cannot talk to daemon 2023-09-22T12:42:21.275 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-22T12:42:21.286 INFO:teuthology.orchestra.run.smithi174.stderr:sudo: ntpd: command not found 2023-09-22T12:42:21.288 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-22T12:42:21.298 INFO:teuthology.orchestra.run.smithi174.stdout:506 Cannot talk to daemon 2023-09-22T12:42:21.312 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-22T12:42:21.326 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-22T12:42:21.364 INFO:teuthology.orchestra.run.smithi069.stderr:bash: ntpq: command not found 2023-09-22T12:42:21.368 INFO:teuthology.orchestra.run.smithi069.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-22T12:42:21.368 INFO:teuthology.orchestra.run.smithi069.stdout:=============================================================================== 2023-09-22T12:42:21.368 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:42:21.368 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:42:21.368 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:42:21.369 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:42:21.392 INFO:teuthology.orchestra.run.smithi174.stderr:bash: ntpq: command not found 2023-09-22T12:42:21.396 INFO:teuthology.orchestra.run.smithi174.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-22T12:42:21.396 INFO:teuthology.orchestra.run.smithi174.stdout:=============================================================================== 2023-09-22T12:42:21.396 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:42:21.396 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:42:21.396 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:42:21.396 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:42:21.397 INFO:teuthology.run_tasks:Running task install... 2023-09-22T12:42:21.405 DEBUG:teuthology.task.install:project ceph 2023-09-22T12:42:21.405 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}} 2023-09-22T12:42:21.405 DEBUG:teuthology.task.install:config {'flavor': 'default', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'} 2023-09-22T12:42:21.405 INFO:teuthology.task.install:Using flavor: default 2023-09-22T12:42:21.410 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-22T12:42:21.410 INFO:teuthology.task.install:extra packages: [] 2023-09-22T12:42:21.411 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'tag': None, 'wait_for_package': False} 2023-09-22T12:42:21.411 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T12:42:21.413 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'tag': None, 'wait_for_package': False} 2023-09-22T12:42:21.413 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T12:42:21.575 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/ 2023-09-22T12:42:21.575 INFO:teuthology.task.install.rpm:Package version is 16.2.14-5.g31c126b7 2023-09-22T12:42:21.690 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/ 2023-09-22T12:42:21.690 INFO:teuthology.task.install.rpm:Package version is 16.2.14-5.g31c126b7 2023-09-22T12:42:21.793 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-22T12:42:21.793 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T12:42:21.793 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-22T12:42:21.795 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-22T12:42:21.795 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T12:42:21.795 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-22T12:42:21.823 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-22T12:42:21.824 DEBUG:teuthology.orchestra.run.smithi174:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-22T12:42:21.824 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-22T12:42:21.825 DEBUG:teuthology.orchestra.run.smithi069:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-22T12:42:21.892 DEBUG:teuthology.orchestra.run.smithi174:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-22T12:42:21.894 DEBUG:teuthology.orchestra.run.smithi069:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-22T12:42:21.970 DEBUG:teuthology.orchestra.run.smithi174:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-22T12:42:21.972 DEBUG:teuthology.orchestra.run.smithi069:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-22T12:42:22.037 INFO:teuthology.orchestra.run.smithi174.stdout:check_obsoletes = 1 2023-09-22T12:42:22.038 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum clean all 2023-09-22T12:42:22.038 INFO:teuthology.orchestra.run.smithi069.stdout:check_obsoletes = 1 2023-09-22T12:42:22.040 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum clean all 2023-09-22T12:42:22.313 INFO:teuthology.orchestra.run.smithi174.stdout:76 files removed 2023-09-22T12:42:22.314 INFO:teuthology.orchestra.run.smithi069.stdout:76 files removed 2023-09-22T12:42:22.341 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install ceph-radosgw 2023-09-22T12:42:22.342 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install ceph-radosgw 2023-09-22T12:42:23.170 INFO:teuthology.orchestra.run.smithi174.stdout:ceph packages for x86_64 183 kB/s | 76 kB 00:00 2023-09-22T12:42:23.208 INFO:teuthology.orchestra.run.smithi069.stdout:ceph packages for x86_64 166 kB/s | 76 kB 00:00 2023-09-22T12:42:23.579 INFO:teuthology.orchestra.run.smithi174.stdout:ceph noarch packages 41 kB/s | 15 kB 00:00 2023-09-22T12:42:23.784 INFO:teuthology.orchestra.run.smithi069.stdout:ceph noarch packages 29 kB/s | 15 kB 00:00 2023-09-22T12:42:24.187 INFO:teuthology.orchestra.run.smithi174.stdout:ceph source packages 2.5 kB/s | 1.4 kB 00:00 2023-09-22T12:42:24.343 INFO:teuthology.orchestra.run.smithi069.stdout:ceph source packages 2.7 kB/s | 1.4 kB 00:00 2023-09-22T13:03:20.048 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - AppStream 27 kB/s | 33 MB 20:55 2023-09-22T13:04:37.285 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 8 - AppStream 26 kB/s | 33 MB 22:13 2023-09-22T13:05:50.556 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - BaseOS 330 kB/s | 47 MB 02:24 2023-09-22T13:05:57.644 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - Extras 45 kB/s | 18 kB 00:00 2023-09-22T13:05:58.169 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - Extras common packages 20 kB/s | 6.9 kB 00:00 2023-09-22T13:05:59.770 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - PowerTools 4.5 MB/s | 6.5 MB 00:01 2023-09-22T13:06:01.280 INFO:teuthology.orchestra.run.smithi069.stdout:Copr repo for python3-asyncssh owned by ceph 14 kB/s | 3.5 kB 00:00 2023-09-22T13:06:03.224 INFO:teuthology.orchestra.run.smithi069.stdout:Extra Packages for Enterprise Linux 8.9 MB/s | 16 MB 00:01 2023-09-22T13:06:07.089 INFO:teuthology.orchestra.run.smithi069.stdout:lab-extras 178 kB/s | 24 kB 00:00 2023-09-22T13:06:11.489 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:06:11.492 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:06:11.492 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repository Size 2023-09-22T13:06:11.492 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:06:11.492 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:06:11.492 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-radosgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 12 M 2023-09-22T13:06:11.492 INFO:teuthology.orchestra.run.smithi069.stdout:Upgrading: 2023-09-22T13:06:11.492 INFO:teuthology.orchestra.run.smithi069.stdout: librados2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T13:06:11.492 INFO:teuthology.orchestra.run.smithi069.stdout: librbd1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 4.2 M 2023-09-22T13:06:11.492 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2023-09-22T13:06:11.492 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-base x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.1 M 2023-09-22T13:06:11.492 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 24 M 2023-09-22T13:06:11.493 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-selinux x86_64 2:16.2.14-5.g31c126b7.el8 ceph 26 k 2023-09-22T13:06:11.493 INFO:teuthology.orchestra.run.smithi069.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-22T13:06:11.493 INFO:teuthology.orchestra.run.smithi069.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-09-22T13:06:11.493 INFO:teuthology.orchestra.run.smithi069.stdout: libcephfs2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 824 k 2023-09-22T13:06:11.493 INFO:teuthology.orchestra.run.smithi069.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-22T13:06:11.493 INFO:teuthology.orchestra.run.smithi069.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-09-22T13:06:11.493 INFO:teuthology.orchestra.run.smithi069.stdout: libradosstriper1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 494 k 2023-09-22T13:06:11.493 INFO:teuthology.orchestra.run.smithi069.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-09-22T13:06:11.493 INFO:teuthology.orchestra.run.smithi069.stdout: librgw2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.8 M 2023-09-22T13:06:11.493 INFO:teuthology.orchestra.run.smithi069.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-22T13:06:11.493 INFO:teuthology.orchestra.run.smithi069.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout: python3-ceph-argparse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 45 k 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout: python3-ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 99 k 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cephfs x86_64 2:16.2.14-5.g31c126b7.el8 ceph 215 k 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout: python3-rados x86_64 2:16.2.14-5.g31c126b7.el8 ceph 390 k 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout: python3-rbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 369 k 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout: python3-rgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 114 k 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout:Install 21 Packages 2023-09-22T13:06:11.494 INFO:teuthology.orchestra.run.smithi069.stdout:Upgrade 2 Packages 2023-09-22T13:06:11.495 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:06:11.495 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 57 M 2023-09-22T13:06:11.495 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:06:14.823 INFO:teuthology.orchestra.run.smithi069.stdout:(1/23): ceph-base-16.2.14-5.g31c126b7.el8.x86_6 2.3 MB/s | 6.1 MB 00:02 2023-09-22T13:06:14.906 INFO:teuthology.orchestra.run.smithi069.stdout:(2/23): ceph-selinux-16.2.14-5.g31c126b7.el8.x8 308 kB/s | 26 kB 00:00 2023-09-22T13:06:15.065 INFO:teuthology.orchestra.run.smithi069.stdout:(3/23): libcephfs2-16.2.14-5.g31c126b7.el8.x86_ 5.1 MB/s | 824 kB 00:00 2023-09-22T13:06:15.198 INFO:teuthology.orchestra.run.smithi069.stdout:(4/23): libradosstriper1-16.2.14-5.g31c126b7.el 3.6 MB/s | 494 kB 00:00 2023-09-22T13:06:15.507 INFO:teuthology.orchestra.run.smithi069.stdout:(5/23): ceph-radosgw-16.2.14-5.g31c126b7.el8.x8 3.5 MB/s | 12 MB 00:03 2023-09-22T13:06:15.607 INFO:teuthology.orchestra.run.smithi069.stdout:(6/23): python3-ceph-argparse-16.2.14-5.g31c126 451 kB/s | 45 kB 00:00 2023-09-22T13:06:15.733 INFO:teuthology.orchestra.run.smithi069.stdout:(7/23): python3-ceph-common-16.2.14-5.g31c126b7 789 kB/s | 99 kB 00:00 2023-09-22T13:06:15.850 INFO:teuthology.orchestra.run.smithi069.stdout:(8/23): python3-cephfs-16.2.14-5.g31c126b7.el8. 1.8 MB/s | 215 kB 00:00 2023-09-22T13:06:15.908 INFO:teuthology.orchestra.run.smithi069.stdout:(9/23): librgw2-16.2.14-5.g31c126b7.el8.x86_64. 5.3 MB/s | 3.8 MB 00:00 2023-09-22T13:06:15.950 INFO:teuthology.orchestra.run.smithi069.stdout:(10/23): python3-rados-16.2.14-5.g31c126b7.el8. 3.8 MB/s | 390 kB 00:00 2023-09-22T13:06:16.001 INFO:teuthology.orchestra.run.smithi069.stdout:(11/23): python3-rbd-16.2.14-5.g31c126b7.el8.x8 3.9 MB/s | 369 kB 00:00 2023-09-22T13:06:16.034 INFO:teuthology.orchestra.run.smithi069.stdout:(12/23): python3-rgw-16.2.14-5.g31c126b7.el8.x8 1.3 MB/s | 114 kB 00:00 2023-09-22T13:06:16.510 INFO:teuthology.orchestra.run.smithi069.stdout:(13/23): ceph-common-16.2.14-5.g31c126b7.el8.x8 5.6 MB/s | 24 MB 00:04 2023-09-22T13:06:18.226 INFO:teuthology.orchestra.run.smithi069.stdout:(14/23): python3-prettytable-0.7.2-14.el8.noarc 26 kB/s | 44 kB 00:01 2023-09-22T13:06:28.800 INFO:teuthology.orchestra.run.smithi069.stdout:(15/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 20 kB/s | 259 kB 00:12 2023-09-22T13:06:29.400 INFO:teuthology.orchestra.run.smithi069.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 510 kB/s | 306 kB 00:00 2023-09-22T13:06:29.534 INFO:teuthology.orchestra.run.smithi069.stdout:(17/23): leveldb-1.22-1.el8.x86_64.rpm 1.3 MB/s | 181 kB 00:00 2023-09-22T13:06:29.651 INFO:teuthology.orchestra.run.smithi069.stdout:(18/23): liboath-2.6.2-3.el8.x86_64.rpm 504 kB/s | 59 kB 00:00 2023-09-22T13:06:29.759 INFO:teuthology.orchestra.run.smithi069.stdout:(19/23): libunwind-1.3.1-3.el8.x86_64.rpm 692 kB/s | 75 kB 00:00 2023-09-22T13:06:30.118 INFO:teuthology.orchestra.run.smithi069.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 25 kB/s | 354 kB 00:14 2023-09-22T13:06:30.344 INFO:teuthology.orchestra.run.smithi069.stdout:(21/23): librados2-16.2.14-5.g31c126b7.el8.x86_ 6.4 MB/s | 3.7 MB 00:00 2023-09-22T13:06:30.910 INFO:teuthology.orchestra.run.smithi069.stdout:(22/23): librbd1-16.2.14-5.g31c126b7.el8.x86_64 5.3 MB/s | 4.2 MB 00:00 2023-09-22T13:07:19.386 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 8 - BaseOS 306 kB/s | 47 MB 02:36 2023-09-22T13:07:22.870 INFO:teuthology.orchestra.run.smithi069.stdout:(23/23): librabbitmq-0.9.0-4.el8.x86_64.rpm 743 B/s | 47 kB 01:04 2023-09-22T13:07:22.873 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:07:22.873 INFO:teuthology.orchestra.run.smithi069.stdout:Total 823 kB/s | 57 MB 01:11 2023-09-22T13:07:22.912 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:07:22.959 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:07:22.959 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:07:23.290 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:07:23.293 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:07:24.313 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:07:24.522 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2023-09-22T13:07:24.710 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T13:07:25.093 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T13:07:25.276 INFO:teuthology.orchestra.run.smithi069.stdout: Upgrading : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T13:07:25.363 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T13:07:25.624 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T13:07:25.793 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T13:07:26.208 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T13:07:26.376 INFO:teuthology.orchestra.run.smithi069.stdout: Upgrading : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T13:07:26.425 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T13:07:26.607 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 7/25 2023-09-22T13:07:26.627 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 8 - Extras 34 kB/s | 18 kB 00:00 2023-09-22T13:07:26.776 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T13:07:27.011 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 8 - Extras common packages 33 kB/s | 6.9 kB 00:00 2023-09-22T13:07:27.134 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T13:07:27.318 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T13:07:27.383 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T13:07:27.582 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-09-22T13:07:27.765 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 11/25 2023-09-22T13:07:27.942 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T13:07:28.160 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T13:07:28.376 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 14/25 2023-09-22T13:07:28.517 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T13:07:28.586 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T13:07:28.801 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-22T13:07:28.967 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2023-09-22T13:07:29.145 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2023-09-22T13:07:29.318 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 19/25 2023-09-22T13:07:31.726 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T13:07:31.951 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T13:07:32.671 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T13:07:32.928 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 8 - PowerTools 1.1 MB/s | 6.5 MB 00:05 2023-09-22T13:07:32.963 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T13:07:33.068 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T13:07:33.208 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T13:07:34.514 INFO:teuthology.orchestra.run.smithi174.stdout:Copr repo for python3-asyncssh owned by ceph 11 kB/s | 3.5 kB 00:00 2023-09-22T13:07:36.497 INFO:teuthology.orchestra.run.smithi174.stdout:Extra Packages for Enterprise Linux 8.7 MB/s | 16 MB 00:01 2023-09-22T13:07:40.380 INFO:teuthology.orchestra.run.smithi174.stdout:lab-extras 185 kB/s | 24 kB 00:00 2023-09-22T13:07:44.830 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:07:44.832 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:07:44.832 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repository Size 2023-09-22T13:07:44.833 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:07:44.833 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:07:44.833 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-radosgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 12 M 2023-09-22T13:07:44.833 INFO:teuthology.orchestra.run.smithi174.stdout:Upgrading: 2023-09-22T13:07:44.833 INFO:teuthology.orchestra.run.smithi174.stdout: librados2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T13:07:44.833 INFO:teuthology.orchestra.run.smithi174.stdout: librbd1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 4.2 M 2023-09-22T13:07:44.834 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2023-09-22T13:07:44.834 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-base x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.1 M 2023-09-22T13:07:44.834 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 24 M 2023-09-22T13:07:44.834 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-selinux x86_64 2:16.2.14-5.g31c126b7.el8 ceph 26 k 2023-09-22T13:07:44.834 INFO:teuthology.orchestra.run.smithi174.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-22T13:07:44.834 INFO:teuthology.orchestra.run.smithi174.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-09-22T13:07:44.834 INFO:teuthology.orchestra.run.smithi174.stdout: libcephfs2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 824 k 2023-09-22T13:07:44.834 INFO:teuthology.orchestra.run.smithi174.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-22T13:07:44.835 INFO:teuthology.orchestra.run.smithi174.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-09-22T13:07:44.835 INFO:teuthology.orchestra.run.smithi174.stdout: libradosstriper1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 494 k 2023-09-22T13:07:44.835 INFO:teuthology.orchestra.run.smithi174.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-09-22T13:07:44.835 INFO:teuthology.orchestra.run.smithi174.stdout: librgw2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.8 M 2023-09-22T13:07:44.835 INFO:teuthology.orchestra.run.smithi174.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-22T13:07:44.835 INFO:teuthology.orchestra.run.smithi174.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-09-22T13:07:44.835 INFO:teuthology.orchestra.run.smithi174.stdout: python3-ceph-argparse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 45 k 2023-09-22T13:07:44.835 INFO:teuthology.orchestra.run.smithi174.stdout: python3-ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 99 k 2023-09-22T13:07:44.836 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cephfs x86_64 2:16.2.14-5.g31c126b7.el8 ceph 215 k 2023-09-22T13:07:44.836 INFO:teuthology.orchestra.run.smithi174.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-09-22T13:07:44.836 INFO:teuthology.orchestra.run.smithi174.stdout: python3-rados x86_64 2:16.2.14-5.g31c126b7.el8 ceph 390 k 2023-09-22T13:07:44.836 INFO:teuthology.orchestra.run.smithi174.stdout: python3-rbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 369 k 2023-09-22T13:07:44.836 INFO:teuthology.orchestra.run.smithi174.stdout: python3-rgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 114 k 2023-09-22T13:07:44.836 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:07:44.836 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:07:44.836 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:07:44.837 INFO:teuthology.orchestra.run.smithi174.stdout:Install 21 Packages 2023-09-22T13:07:44.837 INFO:teuthology.orchestra.run.smithi174.stdout:Upgrade 2 Packages 2023-09-22T13:07:44.837 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:07:44.837 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 57 M 2023-09-22T13:07:44.838 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:07:46.756 INFO:teuthology.orchestra.run.smithi174.stdout:(1/23): ceph-base-16.2.14-5.g31c126b7.el8.x86_6 4.2 MB/s | 6.1 MB 00:01 2023-09-22T13:07:46.981 INFO:teuthology.orchestra.run.smithi174.stdout:(2/23): ceph-common-16.2.14-5.g31c126b7.el8.x86 14 MB/s | 24 MB 00:01 2023-09-22T13:07:47.006 INFO:teuthology.orchestra.run.smithi174.stdout:(3/23): ceph-selinux-16.2.14-5.g31c126b7.el8.x8 102 kB/s | 26 kB 00:00 2023-09-22T13:07:47.056 INFO:teuthology.orchestra.run.smithi174.stdout:(4/23): libcephfs2-16.2.14-5.g31c126b7.el8.x86_ 11 MB/s | 824 kB 00:00 2023-09-22T13:07:47.107 INFO:teuthology.orchestra.run.smithi174.stdout:(5/23): libradosstriper1-16.2.14-5.g31c126b7.el 4.8 MB/s | 494 kB 00:00 2023-09-22T13:07:47.240 INFO:teuthology.orchestra.run.smithi174.stdout:(6/23): python3-ceph-argparse-16.2.14-5.g31c126 337 kB/s | 45 kB 00:00 2023-09-22T13:07:47.299 INFO:teuthology.orchestra.run.smithi174.stdout:(7/23): librgw2-16.2.14-5.g31c126b7.el8.x86_64. 16 MB/s | 3.8 MB 00:00 2023-09-22T13:07:47.324 INFO:teuthology.orchestra.run.smithi174.stdout:(8/23): python3-ceph-common-16.2.14-5.g31c126b7 1.2 MB/s | 99 kB 00:00 2023-09-22T13:07:47.349 INFO:teuthology.orchestra.run.smithi174.stdout:(9/23): python3-cephfs-16.2.14-5.g31c126b7.el8. 4.2 MB/s | 215 kB 00:00 2023-09-22T13:07:47.408 INFO:teuthology.orchestra.run.smithi174.stdout:(10/23): python3-rbd-16.2.14-5.g31c126b7.el8.x8 6.2 MB/s | 369 kB 00:00 2023-09-22T13:07:47.433 INFO:teuthology.orchestra.run.smithi174.stdout:(11/23): python3-rados-16.2.14-5.g31c126b7.el8. 3.5 MB/s | 390 kB 00:00 2023-09-22T13:07:47.459 INFO:teuthology.orchestra.run.smithi174.stdout:(12/23): python3-rgw-16.2.14-5.g31c126b7.el8.x8 2.2 MB/s | 114 kB 00:00 2023-09-22T13:07:47.776 INFO:teuthology.orchestra.run.smithi174.stdout:(13/23): ceph-radosgw-16.2.14-5.g31c126b7.el8.x 4.7 MB/s | 12 MB 00:02 2023-09-22T13:07:50.084 INFO:teuthology.orchestra.run.smithi174.stdout:(14/23): python3-prettytable-0.7.2-14.el8.noarc 19 kB/s | 44 kB 00:02 2023-09-22T13:07:50.626 INFO:teuthology.orchestra.run.smithi174.stdout:(15/23): librabbitmq-0.9.0-4.el8.x86_64.rpm 87 kB/s | 47 kB 00:00 2023-09-22T13:07:51.035 INFO:teuthology.orchestra.run.smithi174.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 750 kB/s | 306 kB 00:00 2023-09-22T13:07:51.099 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T13:07:51.099 INFO:teuthology.orchestra.run.smithi069.stdout:skipping the directory /sys 2023-09-22T13:07:51.100 INFO:teuthology.orchestra.run.smithi069.stdout:skipping the directory /proc 2023-09-22T13:07:51.100 INFO:teuthology.orchestra.run.smithi069.stdout:skipping the directory /dev 2023-09-22T13:07:51.100 INFO:teuthology.orchestra.run.smithi069.stdout:skipping the directory /run 2023-09-22T13:07:51.100 INFO:teuthology.orchestra.run.smithi069.stdout:skipping the directory /mnt 2023-09-22T13:07:51.100 INFO:teuthology.orchestra.run.smithi069.stdout:skipping the directory /var/tmp 2023-09-22T13:07:51.100 INFO:teuthology.orchestra.run.smithi069.stdout:skipping the directory /home 2023-09-22T13:07:51.101 INFO:teuthology.orchestra.run.smithi069.stdout:skipping the directory /tmp 2023-09-22T13:07:51.101 INFO:teuthology.orchestra.run.smithi069.stdout:skipping the directory /dev 2023-09-22T13:07:51.101 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:07:51.118 INFO:teuthology.orchestra.run.smithi174.stdout:(17/23): leveldb-1.22-1.el8.x86_64.rpm 2.1 MB/s | 181 kB 00:00 2023-09-22T13:07:51.185 INFO:teuthology.orchestra.run.smithi174.stdout:(18/23): liboath-2.6.2-3.el8.x86_64.rpm 882 kB/s | 59 kB 00:00 2023-09-22T13:07:51.260 INFO:teuthology.orchestra.run.smithi174.stdout:(19/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.0 MB/s | 75 kB 00:00 2023-09-22T13:07:51.794 INFO:teuthology.orchestra.run.smithi174.stdout:(20/23): librados2-16.2.14-5.g31c126b7.el8.x86_ 7.0 MB/s | 3.7 MB 00:00 2023-09-22T13:07:52.086 INFO:teuthology.orchestra.run.smithi174.stdout:(21/23): librbd1-16.2.14-5.g31c126b7.el8.x86_64 14 MB/s | 4.2 MB 00:00 2023-09-22T13:07:52.260 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T13:07:52.317 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T13:07:52.317 INFO:teuthology.orchestra.run.smithi069.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T13:07:52.507 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T13:07:52.508 INFO:teuthology.orchestra.run.smithi069.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T13:07:53.219 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T13:07:53.219 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 1/25 2023-09-22T13:07:53.220 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2/25 2023-09-22T13:07:53.220 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T13:07:53.220 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T13:07:53.220 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T13:07:53.220 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T13:07:53.220 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 7/25 2023-09-22T13:07:53.220 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 8/25 2023-09-22T13:07:53.220 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 9/25 2023-09-22T13:07:53.220 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 10/25 2023-09-22T13:07:53.220 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 11/25 2023-09-22T13:07:53.220 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T13:07:53.221 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T13:07:53.221 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2023-09-22T13:07:53.221 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2023-09-22T13:07:53.221 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-22T13:07:53.221 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 17/25 2023-09-22T13:07:53.221 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-22T13:07:53.221 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : leveldb-1.22-1.el8.x86_64 19/25 2023-09-22T13:07:53.221 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 20/25 2023-09-22T13:07:53.221 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 21/25 2023-09-22T13:07:53.221 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T13:07:53.222 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-22T13:07:53.222 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 24/25 2023-09-22T13:07:54.324 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T13:07:54.325 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:07:54.325 INFO:teuthology.orchestra.run.smithi069.stdout:Upgraded: 2023-09-22T13:07:54.325 INFO:teuthology.orchestra.run.smithi069.stdout: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.325 INFO:teuthology.orchestra.run.smithi069.stdout: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.325 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:07:54.325 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.325 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.325 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.325 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.325 INFO:teuthology.orchestra.run.smithi069.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-22T13:07:54.325 INFO:teuthology.orchestra.run.smithi069.stdout: leveldb-1.22-1.el8.x86_64 2023-09-22T13:07:54.326 INFO:teuthology.orchestra.run.smithi069.stdout: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.326 INFO:teuthology.orchestra.run.smithi069.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-22T13:07:54.326 INFO:teuthology.orchestra.run.smithi069.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-09-22T13:07:54.326 INFO:teuthology.orchestra.run.smithi069.stdout: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.326 INFO:teuthology.orchestra.run.smithi069.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-22T13:07:54.326 INFO:teuthology.orchestra.run.smithi069.stdout: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.326 INFO:teuthology.orchestra.run.smithi069.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-22T13:07:54.326 INFO:teuthology.orchestra.run.smithi069.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-22T13:07:54.326 INFO:teuthology.orchestra.run.smithi069.stdout: python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.326 INFO:teuthology.orchestra.run.smithi069.stdout: python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.326 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.327 INFO:teuthology.orchestra.run.smithi069.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-22T13:07:54.327 INFO:teuthology.orchestra.run.smithi069.stdout: python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.327 INFO:teuthology.orchestra.run.smithi069.stdout: python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.327 INFO:teuthology.orchestra.run.smithi069.stdout: python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:54.327 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:07:54.327 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:07:54.505 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install ceph-test 2023-09-22T13:07:55.349 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:01:49 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:07:56.936 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:07:56.937 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:07:56.938 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repository Size 2023-09-22T13:07:56.938 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:07:56.938 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:07:56.938 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-test x86_64 2:16.2.14-5.g31c126b7.el8 ceph 52 M 2023-09-22T13:07:56.938 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2023-09-22T13:07:56.938 INFO:teuthology.orchestra.run.smithi069.stdout: jq x86_64 1.6-7.el8 appstream 202 k 2023-09-22T13:07:56.938 INFO:teuthology.orchestra.run.smithi069.stdout: libcephsqlite x86_64 2:16.2.14-5.g31c126b7.el8 ceph 174 k 2023-09-22T13:07:56.939 INFO:teuthology.orchestra.run.smithi069.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-09-22T13:07:56.939 INFO:teuthology.orchestra.run.smithi069.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-09-22T13:07:56.939 INFO:teuthology.orchestra.run.smithi069.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-09-22T13:07:56.939 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:07:56.939 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:07:56.939 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:07:56.939 INFO:teuthology.orchestra.run.smithi069.stdout:Install 6 Packages 2023-09-22T13:07:56.939 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:07:56.940 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 53 M 2023-09-22T13:07:56.940 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 196 M 2023-09-22T13:07:56.940 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:07:57.166 INFO:teuthology.orchestra.run.smithi069.stdout:(1/6): libcephsqlite-16.2.14-5.g31c126b7.el8.x8 945 kB/s | 174 kB 00:00 2023-09-22T13:08:00.467 INFO:teuthology.orchestra.run.smithi069.stdout:(2/6): ceph-test-16.2.14-5.g31c126b7.el8.x86_64 15 MB/s | 52 MB 00:03 2023-09-22T13:08:02.500 INFO:teuthology.orchestra.run.smithi069.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 35 kB/s | 187 kB 00:05 2023-09-22T13:08:04.727 INFO:teuthology.orchestra.run.smithi174.stdout:(22/23): librdkafka-0.11.4-3.el8.x86_64.rpm 20 kB/s | 354 kB 00:17 2023-09-22T13:08:05.941 INFO:teuthology.orchestra.run.smithi069.stdout:(4/6): jq-1.6-7.el8.x86_64.rpm 23 kB/s | 202 kB 00:08 2023-09-22T13:08:09.591 INFO:teuthology.orchestra.run.smithi069.stdout:(5/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 10 kB/s | 71 kB 00:07 2023-09-22T13:08:09.757 INFO:teuthology.orchestra.run.smithi069.stdout:(6/6): socat-1.7.4.1-1.el8.x86_64.rpm 35 kB/s | 323 kB 00:09 2023-09-22T13:08:09.760 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:08:09.760 INFO:teuthology.orchestra.run.smithi069.stdout:Total 4.1 MB/s | 53 MB 00:12 2023-09-22T13:08:09.803 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:08:09.824 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:08:09.824 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:08:10.130 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:08:10.133 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:08:10.612 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:08:10.844 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-09-22T13:08:11.055 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-09-22T13:08:11.186 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T13:08:11.271 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T13:08:11.467 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : jq-1.6-7.el8.x86_64 4/6 2023-09-22T13:08:11.602 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T13:08:16.288 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T13:08:16.508 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T13:08:16.803 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T13:08:16.803 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 1/6 2023-09-22T13:08:16.803 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 2/6 2023-09-22T13:08:16.804 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : jq-1.6-7.el8.x86_64 3/6 2023-09-22T13:08:16.804 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-09-22T13:08:16.804 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-09-22T13:08:17.443 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-09-22T13:08:17.443 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:08:17.443 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:08:17.443 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 jq-1.6-7.el8.x86_64 2023-09-22T13:08:17.443 INFO:teuthology.orchestra.run.smithi069.stdout: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 oniguruma-6.8.2-2.el8.x86_64 2023-09-22T13:08:17.444 INFO:teuthology.orchestra.run.smithi069.stdout: socat-1.7.4.1-1.el8.x86_64 xmlstarlet-1.6.1-20.el8.x86_64 2023-09-22T13:08:17.444 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:08:17.444 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:08:17.659 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install ceph 2023-09-22T13:08:17.842 INFO:teuthology.orchestra.run.smithi174.stdout:(23/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 8.5 kB/s | 259 kB 00:30 2023-09-22T13:08:17.845 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:08:17.845 INFO:teuthology.orchestra.run.smithi174.stdout:Total 1.7 MB/s | 57 MB 00:33 2023-09-22T13:08:17.887 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:08:17.934 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:08:17.934 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:08:18.269 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:08:18.272 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:08:18.520 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:02:12 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:08:19.285 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:08:19.502 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2023-09-22T13:08:19.663 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T13:08:20.048 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T13:08:20.081 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:08:20.084 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:08:20.084 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repository Size 2023-09-22T13:08:20.084 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:08:20.085 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:08:20.085 INFO:teuthology.orchestra.run.smithi069.stdout: ceph x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.4 k 2023-09-22T13:08:20.085 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2023-09-22T13:08:20.085 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mds x86_64 2:16.2.14-5.g31c126b7.el8 ceph 2.4 M 2023-09-22T13:08:20.085 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr x86_64 2:16.2.14-5.g31c126b7.el8 ceph 1.6 M 2023-09-22T13:08:20.085 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr-modules-core noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 244 k 2023-09-22T13:08:20.085 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mon x86_64 2:16.2.14-5.g31c126b7.el8 ceph 5.0 M 2023-09-22T13:08:20.086 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-osd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 18 M 2023-09-22T13:08:20.086 INFO:teuthology.orchestra.run.smithi069.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-22T13:08:20.086 INFO:teuthology.orchestra.run.smithi069.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-22T13:08:20.086 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-09-22T13:08:20.086 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-22T13:08:20.086 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-22T13:08:20.086 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-09-22T13:08:20.086 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-22T13:08:20.087 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-22T13:08:20.087 INFO:teuthology.orchestra.run.smithi069.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-22T13:08:20.087 INFO:teuthology.orchestra.run.smithi069.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-09-22T13:08:20.087 INFO:teuthology.orchestra.run.smithi069.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-09-22T13:08:20.087 INFO:teuthology.orchestra.run.smithi069.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-22T13:08:20.087 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-22T13:08:20.087 INFO:teuthology.orchestra.run.smithi069.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-22T13:08:20.087 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-09-22T13:08:20.088 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-09-22T13:08:20.088 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-09-22T13:08:20.088 INFO:teuthology.orchestra.run.smithi069.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-22T13:08:20.088 INFO:teuthology.orchestra.run.smithi069.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-22T13:08:20.088 INFO:teuthology.orchestra.run.smithi069.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-22T13:08:20.088 INFO:teuthology.orchestra.run.smithi069.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-22T13:08:20.088 INFO:teuthology.orchestra.run.smithi069.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-22T13:08:20.088 INFO:teuthology.orchestra.run.smithi069.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-22T13:08:20.088 INFO:teuthology.orchestra.run.smithi069.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-22T13:08:20.089 INFO:teuthology.orchestra.run.smithi069.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-09-22T13:08:20.089 INFO:teuthology.orchestra.run.smithi069.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-22T13:08:20.089 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:08:20.089 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:08:20.089 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:08:20.089 INFO:teuthology.orchestra.run.smithi069.stdout:Install 32 Packages 2023-09-22T13:08:20.089 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:08:20.090 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 31 M 2023-09-22T13:08:20.090 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 101 M 2023-09-22T13:08:20.090 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:08:20.230 INFO:teuthology.orchestra.run.smithi174.stdout: Upgrading : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T13:08:20.307 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T13:08:20.500 INFO:teuthology.orchestra.run.smithi069.stdout:(1/32): ceph-16.2.14-5.g31c126b7.el8.x86_64.rpm 24 kB/s | 6.4 kB 00:00 2023-09-22T13:08:20.570 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T13:08:20.634 INFO:teuthology.orchestra.run.smithi069.stdout:(2/32): ceph-mgr-16.2.14-5.g31c126b7.el8.x86_64 4.0 MB/s | 1.6 MB 00:00 2023-09-22T13:08:20.755 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T13:08:20.801 INFO:teuthology.orchestra.run.smithi069.stdout:(3/32): ceph-mds-16.2.14-5.g31c126b7.el8.x86_64 4.3 MB/s | 2.4 MB 00:00 2023-09-22T13:08:20.893 INFO:teuthology.orchestra.run.smithi069.stdout:(4/32): ceph-mgr-modules-core-16.2.14-5.g31c126 2.6 MB/s | 244 kB 00:00 2023-09-22T13:08:21.158 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T13:08:21.338 INFO:teuthology.orchestra.run.smithi174.stdout: Upgrading : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T13:08:21.404 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T13:08:21.585 INFO:teuthology.orchestra.run.smithi069.stdout:(5/32): ceph-osd-16.2.14-5.g31c126b7.el8.x86_64 19 MB/s | 18 MB 00:00 2023-09-22T13:08:21.610 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 7/25 2023-09-22T13:08:21.780 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T13:08:21.901 INFO:teuthology.orchestra.run.smithi069.stdout:(6/32): ceph-mon-16.2.14-5.g31c126b7.el8.x86_64 3.6 MB/s | 5.0 MB 00:01 2023-09-22T13:08:22.155 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T13:08:22.321 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T13:08:22.395 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T13:08:22.593 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-09-22T13:08:22.768 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 11/25 2023-09-22T13:08:22.936 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T13:08:23.147 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T13:08:23.354 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 14/25 2023-09-22T13:08:23.504 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T13:08:23.574 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T13:08:23.763 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-22T13:08:23.962 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2023-09-22T13:08:24.150 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2023-09-22T13:08:24.313 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 19/25 2023-09-22T13:08:24.526 INFO:teuthology.orchestra.run.smithi069.stdout:(7/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 39 kB/s | 103 kB 00:02 2023-09-22T13:08:24.993 INFO:teuthology.orchestra.run.smithi069.stdout:(8/32): python3-markupsafe-0.23-19.el8.x86_64.r 11 kB/s | 39 kB 00:03 2023-09-22T13:08:26.749 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T13:08:26.954 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T13:08:27.650 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T13:08:27.935 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T13:08:28.033 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T13:08:28.061 INFO:teuthology.orchestra.run.smithi069.stdout:(9/32): python3-mako-1.0.6-14.el8.noarch.rpm 22 kB/s | 157 kB 00:07 2023-09-22T13:08:28.298 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T13:08:28.319 INFO:teuthology.orchestra.run.smithi069.stdout:(10/32): python3-pytz-2017.2-11.el8.noarch.rpm 15 kB/s | 57 kB 00:03 2023-09-22T13:08:33.902 INFO:teuthology.orchestra.run.smithi069.stdout:(11/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 43 kB/s | 249 kB 00:05 2023-09-22T13:08:34.377 INFO:teuthology.orchestra.run.smithi069.stdout:(12/32): python3-cryptography-3.2.1-6.el8.x86_6 106 kB/s | 641 kB 00:06 2023-09-22T13:08:34.961 INFO:teuthology.orchestra.run.smithi069.stdout:(13/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 75 kB/s | 44 kB 00:00 2023-09-22T13:08:35.244 INFO:teuthology.orchestra.run.smithi069.stdout:(14/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 654 kB/s | 185 kB 00:00 2023-09-22T13:08:35.395 INFO:teuthology.orchestra.run.smithi069.stdout:(15/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 1.1 MB/s | 173 kB 00:00 2023-09-22T13:08:35.586 INFO:teuthology.orchestra.run.smithi069.stdout:(16/32): python3-cherrypy-18.4.0-1.el8.noarch.r 2.0 MB/s | 384 kB 00:00 2023-09-22T13:08:35.687 INFO:teuthology.orchestra.run.smithi069.stdout:(17/32): python3-jaraco-6.2-6.el8.noarch.rpm 111 kB/s | 11 kB 00:00 2023-09-22T13:08:35.795 INFO:teuthology.orchestra.run.smithi069.stdout:(18/32): python3-jaraco-functools-2.0-4.el8.noa 162 kB/s | 18 kB 00:00 2023-09-22T13:08:35.812 INFO:teuthology.orchestra.run.smithi069.stdout:(19/32): python3-pycparser-2.14-14.el8.noarch.r 57 kB/s | 109 kB 00:01 2023-09-22T13:08:35.904 INFO:teuthology.orchestra.run.smithi069.stdout:(20/32): python3-logutils-0.3.5-11.el8.noarch.r 449 kB/s | 49 kB 00:00 2023-09-22T13:08:36.030 INFO:teuthology.orchestra.run.smithi069.stdout:(21/32): python3-pecan-1.3.2-9.el8.noarch.rpm 2.3 MB/s | 283 kB 00:00 2023-09-22T13:08:36.130 INFO:teuthology.orchestra.run.smithi069.stdout:(22/32): python3-portend-2.6-1.el8.noarch.rpm 156 kB/s | 16 kB 00:00 2023-09-22T13:08:36.238 INFO:teuthology.orchestra.run.smithi069.stdout:(23/32): python3-simplegeneric-0.8.1-17.el8.noa 173 kB/s | 19 kB 00:00 2023-09-22T13:08:36.272 INFO:teuthology.orchestra.run.smithi069.stdout:(24/32): python3-more-itertools-7.2.0-3.el8.noa 128 kB/s | 59 kB 00:00 2023-09-22T13:08:36.339 INFO:teuthology.orchestra.run.smithi069.stdout:(25/32): python3-singledispatch-3.4.0.3-18.el8. 238 kB/s | 24 kB 00:00 2023-09-22T13:08:36.397 INFO:teuthology.orchestra.run.smithi069.stdout:(26/32): python3-tempora-1.14.1-5.el8.noarch.rp 233 kB/s | 29 kB 00:00 2023-09-22T13:08:36.431 INFO:teuthology.orchestra.run.smithi069.stdout:(27/32): python3-trustme-0.6.0-4.el8.noarch.rpm 299 kB/s | 27 kB 00:00 2023-09-22T13:08:36.548 INFO:teuthology.orchestra.run.smithi069.stdout:(28/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.1 MB/s | 251 kB 00:00 2023-09-22T13:08:36.657 INFO:teuthology.orchestra.run.smithi069.stdout:(29/32): python3-webtest-2.0.33-1.el8.noarch.rp 789 kB/s | 85 kB 00:00 2023-09-22T13:08:36.674 INFO:teuthology.orchestra.run.smithi069.stdout:(30/32): python3-waitress-1.4.3-1.el8.noarch.rp 925 kB/s | 255 kB 00:00 2023-09-22T13:08:36.757 INFO:teuthology.orchestra.run.smithi069.stdout:(31/32): python3-zc-lockfile-2.0-2.el8.noarch.r 229 kB/s | 23 kB 00:00 2023-09-22T13:08:40.740 INFO:teuthology.orchestra.run.smithi069.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 29 kB/s | 457 kB 00:15 2023-09-22T13:08:40.744 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:08:40.744 INFO:teuthology.orchestra.run.smithi069.stdout:Total 1.5 MB/s | 31 MB 00:20 2023-09-22T13:08:40.829 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:08:40.868 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:08:40.868 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:08:41.183 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:08:41.185 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:08:42.265 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:08:42.536 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-22T13:08:42.738 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-22T13:08:42.955 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-09-22T13:08:43.245 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-09-22T13:08:43.452 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-09-22T13:08:43.649 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-09-22T13:08:43.851 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-09-22T13:08:44.070 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-09-22T13:08:44.279 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-09-22T13:08:44.500 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-09-22T13:08:44.721 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-09-22T13:08:44.928 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-09-22T13:08:45.301 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-09-22T13:08:45.567 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-09-22T13:08:45.794 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-09-22T13:08:46.039 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-09-22T13:08:46.248 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T13:08:46.392 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T13:08:46.392 INFO:teuthology.orchestra.run.smithi174.stdout:skipping the directory /sys 2023-09-22T13:08:46.392 INFO:teuthology.orchestra.run.smithi174.stdout:skipping the directory /proc 2023-09-22T13:08:46.392 INFO:teuthology.orchestra.run.smithi174.stdout:skipping the directory /dev 2023-09-22T13:08:46.393 INFO:teuthology.orchestra.run.smithi174.stdout:skipping the directory /run 2023-09-22T13:08:46.393 INFO:teuthology.orchestra.run.smithi174.stdout:skipping the directory /mnt 2023-09-22T13:08:46.393 INFO:teuthology.orchestra.run.smithi174.stdout:skipping the directory /var/tmp 2023-09-22T13:08:46.393 INFO:teuthology.orchestra.run.smithi174.stdout:skipping the directory /home 2023-09-22T13:08:46.393 INFO:teuthology.orchestra.run.smithi174.stdout:skipping the directory /tmp 2023-09-22T13:08:46.393 INFO:teuthology.orchestra.run.smithi174.stdout:skipping the directory /dev 2023-09-22T13:08:46.393 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:08:46.539 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-09-22T13:08:46.735 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-09-22T13:08:46.924 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-09-22T13:08:47.118 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-22T13:08:47.396 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-22T13:08:47.523 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T13:08:47.601 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T13:08:47.601 INFO:teuthology.orchestra.run.smithi174.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T13:08:47.620 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-22T13:08:47.811 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T13:08:47.811 INFO:teuthology.orchestra.run.smithi174.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T13:08:48.485 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T13:08:48.485 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 1/25 2023-09-22T13:08:48.485 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2/25 2023-09-22T13:08:48.486 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T13:08:48.486 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T13:08:48.486 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T13:08:48.486 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T13:08:48.486 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 7/25 2023-09-22T13:08:48.486 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 8/25 2023-09-22T13:08:48.486 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 9/25 2023-09-22T13:08:48.487 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 10/25 2023-09-22T13:08:48.487 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 11/25 2023-09-22T13:08:48.487 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T13:08:48.487 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T13:08:48.487 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2023-09-22T13:08:48.487 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2023-09-22T13:08:48.487 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-22T13:08:48.488 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 17/25 2023-09-22T13:08:48.488 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-22T13:08:48.488 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : leveldb-1.22-1.el8.x86_64 19/25 2023-09-22T13:08:48.488 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 20/25 2023-09-22T13:08:48.488 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 21/25 2023-09-22T13:08:48.488 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T13:08:48.488 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-22T13:08:48.488 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 24/25 2023-09-22T13:08:48.765 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-09-22T13:08:49.452 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-09-22T13:08:49.541 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T13:08:49.542 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:08:49.542 INFO:teuthology.orchestra.run.smithi174.stdout:Upgraded: 2023-09-22T13:08:49.542 INFO:teuthology.orchestra.run.smithi174.stdout: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.542 INFO:teuthology.orchestra.run.smithi174.stdout: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.542 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:08:49.542 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.542 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.543 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.543 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.543 INFO:teuthology.orchestra.run.smithi174.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-22T13:08:49.543 INFO:teuthology.orchestra.run.smithi174.stdout: leveldb-1.22-1.el8.x86_64 2023-09-22T13:08:49.543 INFO:teuthology.orchestra.run.smithi174.stdout: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.543 INFO:teuthology.orchestra.run.smithi174.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-22T13:08:49.543 INFO:teuthology.orchestra.run.smithi174.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-09-22T13:08:49.544 INFO:teuthology.orchestra.run.smithi174.stdout: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.544 INFO:teuthology.orchestra.run.smithi174.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-22T13:08:49.544 INFO:teuthology.orchestra.run.smithi174.stdout: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.544 INFO:teuthology.orchestra.run.smithi174.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-22T13:08:49.544 INFO:teuthology.orchestra.run.smithi174.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-22T13:08:49.544 INFO:teuthology.orchestra.run.smithi174.stdout: python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.544 INFO:teuthology.orchestra.run.smithi174.stdout: python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.545 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.545 INFO:teuthology.orchestra.run.smithi174.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-22T13:08:49.545 INFO:teuthology.orchestra.run.smithi174.stdout: python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.545 INFO:teuthology.orchestra.run.smithi174.stdout: python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.545 INFO:teuthology.orchestra.run.smithi174.stdout: python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:49.545 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:08:49.545 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:08:49.698 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install ceph-test 2023-09-22T13:08:49.733 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-22T13:08:50.085 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 27/32 2023-09-22T13:08:50.251 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T13:08:50.548 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:01:10 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:08:51.755 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T13:08:51.905 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T13:08:52.118 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:08:52.119 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:08:52.119 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repository Size 2023-09-22T13:08:52.119 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:08:52.119 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:08:52.120 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-test x86_64 2:16.2.14-5.g31c126b7.el8 ceph 52 M 2023-09-22T13:08:52.120 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2023-09-22T13:08:52.120 INFO:teuthology.orchestra.run.smithi174.stdout: jq x86_64 1.6-7.el8 appstream 202 k 2023-09-22T13:08:52.120 INFO:teuthology.orchestra.run.smithi174.stdout: libcephsqlite x86_64 2:16.2.14-5.g31c126b7.el8 ceph 174 k 2023-09-22T13:08:52.120 INFO:teuthology.orchestra.run.smithi174.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-09-22T13:08:52.120 INFO:teuthology.orchestra.run.smithi174.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-09-22T13:08:52.120 INFO:teuthology.orchestra.run.smithi174.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-09-22T13:08:52.120 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:08:52.120 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:08:52.120 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:08:52.120 INFO:teuthology.orchestra.run.smithi174.stdout:Install 6 Packages 2023-09-22T13:08:52.121 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:08:52.121 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 53 M 2023-09-22T13:08:52.121 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 196 M 2023-09-22T13:08:52.121 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:08:52.329 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T13:08:52.512 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T13:08:53.546 INFO:teuthology.orchestra.run.smithi174.stdout:(1/6): libcephsqlite-16.2.14-5.g31c126b7.el8.x8 556 kB/s | 174 kB 00:00 2023-09-22T13:08:53.550 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T13:08:53.550 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T13:08:53.551 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T13:08:53.551 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T13:08:53.551 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T13:08:53.551 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 1/32 2023-09-22T13:08:53.551 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2/32 2023-09-22T13:08:53.551 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 3/32 2023-09-22T13:08:53.551 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 4/32 2023-09-22T13:08:53.551 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 5/32 2023-09-22T13:08:53.552 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 6/32 2023-09-22T13:08:53.552 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-09-22T13:08:53.552 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-09-22T13:08:53.552 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-09-22T13:08:53.552 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-09-22T13:08:53.552 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-09-22T13:08:53.552 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-09-22T13:08:53.552 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-09-22T13:08:53.553 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-09-22T13:08:53.553 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-09-22T13:08:53.553 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-09-22T13:08:53.553 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T13:08:53.553 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-09-22T13:08:53.553 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-09-22T13:08:53.553 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-09-22T13:08:53.554 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-09-22T13:08:53.554 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-09-22T13:08:53.554 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-09-22T13:08:53.554 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-09-22T13:08:53.554 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-09-22T13:08:53.554 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-09-22T13:08:53.554 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-09-22T13:08:53.555 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-09-22T13:08:53.555 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-09-22T13:08:53.555 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-09-22T13:08:53.555 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-09-22T13:08:54.537 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-09-22T13:08:54.617 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:08:54.617 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:08:54.617 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:54.617 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:54.617 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:54.618 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:08:54.618 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:54.618 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:08:54.618 INFO:teuthology.orchestra.run.smithi069.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-22T13:08:54.618 INFO:teuthology.orchestra.run.smithi069.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-22T13:08:54.618 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-09-22T13:08:54.618 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-22T13:08:54.618 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-22T13:08:54.618 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-09-22T13:08:54.618 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-22T13:08:54.618 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-mako-1.0.6-14.el8.noarch 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-portend-2.6-1.el8.noarch 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pytz-2017.2-11.el8.noarch 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-22T13:08:54.619 INFO:teuthology.orchestra.run.smithi069.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-22T13:08:54.620 INFO:teuthology.orchestra.run.smithi069.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-22T13:08:54.620 INFO:teuthology.orchestra.run.smithi069.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-22T13:08:54.620 INFO:teuthology.orchestra.run.smithi069.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-22T13:08:54.620 INFO:teuthology.orchestra.run.smithi069.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-22T13:08:54.620 INFO:teuthology.orchestra.run.smithi069.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-22T13:08:54.620 INFO:teuthology.orchestra.run.smithi069.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-22T13:08:54.620 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:08:54.620 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:08:54.799 INFO:teuthology.orchestra.run.smithi174.stdout:(2/6): ceph-test-16.2.14-5.g31c126b7.el8.x86_64 21 MB/s | 52 MB 00:02 2023-09-22T13:08:54.921 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install ceph-base 2023-09-22T13:08:55.775 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:02:49 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:08:56.512 INFO:teuthology.orchestra.run.smithi174.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 50 kB/s | 187 kB 00:03 2023-09-22T13:08:57.173 INFO:teuthology.orchestra.run.smithi069.stdout:Package ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:08:57.751 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:08:57.751 INFO:teuthology.orchestra.run.smithi069.stdout:Nothing to do. 2023-09-22T13:08:57.751 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:08:57.752 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install cephadm 2023-09-22T13:08:58.475 INFO:teuthology.orchestra.run.smithi174.stdout:(4/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 33 kB/s | 71 kB 00:02 2023-09-22T13:08:58.954 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:02:52 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:09:00.155 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:09:00.156 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:09:00.156 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repository Size 2023-09-22T13:09:00.156 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:09:00.156 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:09:00.156 INFO:teuthology.orchestra.run.smithi069.stdout: cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 84 k 2023-09-22T13:09:00.156 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:00.156 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:09:00.156 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:09:00.156 INFO:teuthology.orchestra.run.smithi069.stdout:Install 1 Package 2023-09-22T13:09:00.156 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:00.157 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 84 k 2023-09-22T13:09:00.157 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 348 k 2023-09-22T13:09:00.157 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:09:00.321 INFO:teuthology.orchestra.run.smithi069.stdout:cephadm-16.2.14-5.g31c126b7.el8.noarch.rpm 513 kB/s | 84 kB 00:00 2023-09-22T13:09:00.321 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:09:00.322 INFO:teuthology.orchestra.run.smithi069.stdout:Total 509 kB/s | 84 kB 00:00 2023-09-22T13:09:00.322 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:09:00.329 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:09:00.329 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:09:00.331 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:09:00.334 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:09:00.378 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:09:00.382 INFO:teuthology.orchestra.run.smithi174.stdout:(5/6): jq-1.6-7.el8.x86_64.rpm 25 kB/s | 202 kB 00:08 2023-09-22T13:09:00.864 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T13:09:01.020 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T13:09:01.253 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T13:09:01.837 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T13:09:01.838 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:01.838 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:09:01.838 INFO:teuthology.orchestra.run.smithi069.stdout: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:09:01.838 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:01.838 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:09:02.039 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install ceph-immutable-object-cache 2023-09-22T13:09:02.891 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:02:56 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:09:04.466 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:09:04.467 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:09:04.467 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repo Size 2023-09-22T13:09:04.467 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:09:04.467 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:09:04.467 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-5.g31c126b7.el8 ceph 161 k 2023-09-22T13:09:04.467 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:04.468 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:09:04.468 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:09:04.468 INFO:teuthology.orchestra.run.smithi069.stdout:Install 1 Package 2023-09-22T13:09:04.468 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:04.468 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 161 k 2023-09-22T13:09:04.469 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 470 k 2023-09-22T13:09:04.469 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:09:04.759 INFO:teuthology.orchestra.run.smithi069.stdout:ceph-immutable-object-cache-16.2.14-5.g31c126b7 555 kB/s | 161 kB 00:00 2023-09-22T13:09:04.759 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:09:04.760 INFO:teuthology.orchestra.run.smithi069.stdout:Total 552 kB/s | 161 kB 00:00 2023-09-22T13:09:04.760 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:09:04.771 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:09:04.771 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:09:04.819 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:09:04.823 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:09:04.948 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:09:05.114 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T13:09:05.527 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T13:09:06.115 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T13:09:06.115 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:06.115 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:09:06.115 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:09:06.116 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:06.116 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:09:06.309 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install ceph-mgr 2023-09-22T13:09:07.168 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:03:01 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:09:08.556 INFO:teuthology.orchestra.run.smithi069.stdout:Package ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:09:08.738 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:09:08.739 INFO:teuthology.orchestra.run.smithi069.stdout:Nothing to do. 2023-09-22T13:09:08.739 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:09:08.914 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install ceph-mgr-dashboard 2023-09-22T13:09:09.772 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:03:03 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:09:11.328 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:09:11.329 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:09:11.329 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repository Size 2023-09-22T13:09:11.329 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:09:11.330 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:09:11.330 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr-dashboard noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 3.1 M 2023-09-22T13:09:11.330 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2023-09-22T13:09:11.330 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-grafana-dashboards noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 24 k 2023-09-22T13:09:11.330 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-prometheus-alerts noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 15 k 2023-09-22T13:09:11.330 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-09-22T13:09:11.330 INFO:teuthology.orchestra.run.smithi069.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-22T13:09:11.330 INFO:teuthology.orchestra.run.smithi069.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-22T13:09:11.330 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:11.331 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:09:11.331 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:09:11.331 INFO:teuthology.orchestra.run.smithi069.stdout:Install 6 Packages 2023-09-22T13:09:11.331 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:11.331 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 3.4 M 2023-09-22T13:09:11.332 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 22 M 2023-09-22T13:09:11.332 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:09:12.153 INFO:teuthology.orchestra.run.smithi069.stdout:(1/6): ceph-grafana-dashboards-16.2.14-5.g31c12 177 kB/s | 24 kB 00:00 2023-09-22T13:09:12.254 INFO:teuthology.orchestra.run.smithi069.stdout:(2/6): ceph-prometheus-alerts-16.2.14-5.g31c126 62 kB/s | 15 kB 00:00 2023-09-22T13:09:12.496 INFO:teuthology.orchestra.run.smithi069.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 139 kB/s | 34 kB 00:00 2023-09-22T13:09:12.738 INFO:teuthology.orchestra.run.smithi069.stdout:(4/6): python3-routes-2.4.1-12.el8.noarch.rpm 813 kB/s | 196 kB 00:00 2023-09-22T13:09:12.804 INFO:teuthology.orchestra.run.smithi069.stdout:(5/6): ceph-mgr-dashboard-16.2.14-5.g31c126b7.e 4.0 MB/s | 3.1 MB 00:00 2023-09-22T13:09:13.448 INFO:teuthology.orchestra.run.smithi174.stdout:(6/6): socat-1.7.4.1-1.el8.x86_64.rpm 17 kB/s | 323 kB 00:18 2023-09-22T13:09:13.450 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:09:13.450 INFO:teuthology.orchestra.run.smithi174.stdout:Total 2.5 MB/s | 53 MB 00:21 2023-09-22T13:09:13.493 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:09:13.514 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:09:13.515 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:09:13.864 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:09:13.867 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:09:14.340 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:09:14.573 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-09-22T13:09:14.782 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-09-22T13:09:14.905 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T13:09:14.970 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T13:09:15.161 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : jq-1.6-7.el8.x86_64 4/6 2023-09-22T13:09:15.296 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T13:09:19.905 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T13:09:20.136 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T13:09:20.421 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T13:09:20.422 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 1/6 2023-09-22T13:09:20.422 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 2/6 2023-09-22T13:09:20.422 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : jq-1.6-7.el8.x86_64 3/6 2023-09-22T13:09:20.422 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-09-22T13:09:20.423 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-09-22T13:09:21.079 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-09-22T13:09:21.079 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:09:21.079 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:09:21.080 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 jq-1.6-7.el8.x86_64 2023-09-22T13:09:21.080 INFO:teuthology.orchestra.run.smithi174.stdout: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 oniguruma-6.8.2-2.el8.x86_64 2023-09-22T13:09:21.080 INFO:teuthology.orchestra.run.smithi174.stdout: socat-1.7.4.1-1.el8.x86_64 xmlstarlet-1.6.1-20.el8.x86_64 2023-09-22T13:09:21.080 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:09:21.080 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:09:21.479 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install ceph 2023-09-22T13:09:22.346 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:01:42 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:09:23.910 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:09:23.914 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:09:23.914 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repository Size 2023-09-22T13:09:23.914 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:09:23.914 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:09:23.914 INFO:teuthology.orchestra.run.smithi174.stdout: ceph x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.4 k 2023-09-22T13:09:23.914 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2023-09-22T13:09:23.915 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mds x86_64 2:16.2.14-5.g31c126b7.el8 ceph 2.4 M 2023-09-22T13:09:23.915 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr x86_64 2:16.2.14-5.g31c126b7.el8 ceph 1.6 M 2023-09-22T13:09:23.915 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr-modules-core noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 244 k 2023-09-22T13:09:23.915 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mon x86_64 2:16.2.14-5.g31c126b7.el8 ceph 5.0 M 2023-09-22T13:09:23.915 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-osd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 18 M 2023-09-22T13:09:23.915 INFO:teuthology.orchestra.run.smithi174.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-22T13:09:23.916 INFO:teuthology.orchestra.run.smithi174.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-22T13:09:23.916 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-09-22T13:09:23.916 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-22T13:09:23.916 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-22T13:09:23.916 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-09-22T13:09:23.916 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-22T13:09:23.916 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-22T13:09:23.916 INFO:teuthology.orchestra.run.smithi174.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-22T13:09:23.917 INFO:teuthology.orchestra.run.smithi174.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-09-22T13:09:23.917 INFO:teuthology.orchestra.run.smithi174.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-09-22T13:09:23.917 INFO:teuthology.orchestra.run.smithi174.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-22T13:09:23.917 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-22T13:09:23.917 INFO:teuthology.orchestra.run.smithi174.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-22T13:09:23.917 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-09-22T13:09:23.917 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-09-22T13:09:23.917 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-09-22T13:09:23.918 INFO:teuthology.orchestra.run.smithi174.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-22T13:09:23.918 INFO:teuthology.orchestra.run.smithi174.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-22T13:09:23.918 INFO:teuthology.orchestra.run.smithi174.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-22T13:09:23.918 INFO:teuthology.orchestra.run.smithi174.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-22T13:09:23.918 INFO:teuthology.orchestra.run.smithi174.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-22T13:09:23.918 INFO:teuthology.orchestra.run.smithi174.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-22T13:09:23.918 INFO:teuthology.orchestra.run.smithi174.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-22T13:09:23.919 INFO:teuthology.orchestra.run.smithi174.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-09-22T13:09:23.919 INFO:teuthology.orchestra.run.smithi174.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-22T13:09:23.919 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:09:23.919 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:09:23.919 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:09:23.919 INFO:teuthology.orchestra.run.smithi174.stdout:Install 32 Packages 2023-09-22T13:09:23.919 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:09:23.920 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 31 M 2023-09-22T13:09:23.920 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 101 M 2023-09-22T13:09:23.920 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:09:24.381 INFO:teuthology.orchestra.run.smithi174.stdout:(1/32): ceph-16.2.14-5.g31c126b7.el8.x86_64.rpm 32 kB/s | 6.4 kB 00:00 2023-09-22T13:09:24.673 INFO:teuthology.orchestra.run.smithi174.stdout:(2/32): ceph-mds-16.2.14-5.g31c126b7.el8.x86_64 5.0 MB/s | 2.4 MB 00:00 2023-09-22T13:09:24.715 INFO:teuthology.orchestra.run.smithi174.stdout:(3/32): ceph-mgr-16.2.14-5.g31c126b7.el8.x86_64 3.0 MB/s | 1.6 MB 00:00 2023-09-22T13:09:24.799 INFO:teuthology.orchestra.run.smithi174.stdout:(4/32): ceph-mgr-modules-core-16.2.14-5.g31c126 2.9 MB/s | 244 kB 00:00 2023-09-22T13:09:25.557 INFO:teuthology.orchestra.run.smithi174.stdout:(5/32): ceph-mon-16.2.14-5.g31c126b7.el8.x86_64 4.3 MB/s | 5.0 MB 00:01 2023-09-22T13:09:25.724 INFO:teuthology.orchestra.run.smithi174.stdout:(6/32): ceph-osd-16.2.14-5.g31c126b7.el8.x86_64 17 MB/s | 18 MB 00:01 2023-09-22T13:09:26.991 INFO:teuthology.orchestra.run.smithi174.stdout:(7/32): python3-markupsafe-0.23-19.el8.x86_64.r 27 kB/s | 39 kB 00:01 2023-09-22T13:09:28.724 INFO:teuthology.orchestra.run.smithi174.stdout:(8/32): python3-pytz-2017.2-11.el8.noarch.rpm 33 kB/s | 57 kB 00:01 2023-09-22T13:09:31.375 INFO:teuthology.orchestra.run.smithi174.stdout:(9/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 18 kB/s | 103 kB 00:05 2023-09-22T13:09:32.341 INFO:teuthology.orchestra.run.smithi174.stdout:(10/32): python3-mako-1.0.6-14.el8.noarch.rpm 21 kB/s | 157 kB 00:07 2023-09-22T13:09:35.691 INFO:teuthology.orchestra.run.smithi174.stdout:(11/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 58 kB/s | 249 kB 00:04 2023-09-22T13:09:41.823 INFO:teuthology.orchestra.run.smithi174.stdout:(12/32): python3-pycparser-2.14-14.el8.noarch.r 18 kB/s | 109 kB 00:06 2023-09-22T13:09:42.141 INFO:teuthology.orchestra.run.smithi174.stdout:(13/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 138 kB/s | 44 kB 00:00 2023-09-22T13:09:42.424 INFO:teuthology.orchestra.run.smithi174.stdout:(14/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 654 kB/s | 185 kB 00:00 2023-09-22T13:09:42.599 INFO:teuthology.orchestra.run.smithi174.stdout:(15/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 986 kB/s | 173 kB 00:00 2023-09-22T13:09:42.941 INFO:teuthology.orchestra.run.smithi174.stdout:(16/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.1 MB/s | 384 kB 00:00 2023-09-22T13:09:43.041 INFO:teuthology.orchestra.run.smithi174.stdout:(17/32): python3-jaraco-6.2-6.el8.noarch.rpm 111 kB/s | 11 kB 00:00 2023-09-22T13:09:43.158 INFO:teuthology.orchestra.run.smithi174.stdout:(18/32): python3-jaraco-functools-2.0-4.el8.noa 151 kB/s | 18 kB 00:00 2023-09-22T13:09:43.275 INFO:teuthology.orchestra.run.smithi174.stdout:(19/32): python3-logutils-0.3.5-11.el8.noarch.r 418 kB/s | 49 kB 00:00 2023-09-22T13:09:43.392 INFO:teuthology.orchestra.run.smithi174.stdout:(20/32): python3-more-itertools-7.2.0-3.el8.noa 503 kB/s | 59 kB 00:00 2023-09-22T13:09:43.692 INFO:teuthology.orchestra.run.smithi174.stdout:(21/32): python3-pecan-1.3.2-9.el8.noarch.rpm 943 kB/s | 283 kB 00:00 2023-09-22T13:09:43.793 INFO:teuthology.orchestra.run.smithi174.stdout:(22/32): python3-portend-2.6-1.el8.noarch.rpm 156 kB/s | 16 kB 00:00 2023-09-22T13:09:43.893 INFO:teuthology.orchestra.run.smithi174.stdout:(23/32): python3-simplegeneric-0.8.1-17.el8.noa 187 kB/s | 19 kB 00:00 2023-09-22T13:09:43.993 INFO:teuthology.orchestra.run.smithi174.stdout:(24/32): python3-singledispatch-3.4.0.3-18.el8. 239 kB/s | 24 kB 00:00 2023-09-22T13:09:44.093 INFO:teuthology.orchestra.run.smithi174.stdout:(25/32): python3-tempora-1.14.1-5.el8.noarch.rp 293 kB/s | 29 kB 00:00 2023-09-22T13:09:44.202 INFO:teuthology.orchestra.run.smithi174.stdout:(26/32): python3-trustme-0.6.0-4.el8.noarch.rpm 254 kB/s | 27 kB 00:00 2023-09-22T13:09:44.377 INFO:teuthology.orchestra.run.smithi174.stdout:(27/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.4 MB/s | 255 kB 00:00 2023-09-22T13:09:44.552 INFO:teuthology.orchestra.run.smithi174.stdout:(28/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.4 MB/s | 251 kB 00:00 2023-09-22T13:09:44.653 INFO:teuthology.orchestra.run.smithi174.stdout:(29/32): python3-webtest-2.0.33-1.el8.noarch.rp 853 kB/s | 85 kB 00:00 2023-09-22T13:09:44.753 INFO:teuthology.orchestra.run.smithi174.stdout:(30/32): python3-zc-lockfile-2.0-2.el8.noarch.r 230 kB/s | 23 kB 00:00 2023-09-22T13:09:45.345 INFO:teuthology.orchestra.run.smithi174.stdout:(31/32): python3-cryptography-3.2.1-6.el8.x86_6 49 kB/s | 641 kB 00:13 2023-09-22T13:09:46.254 INFO:teuthology.orchestra.run.smithi069.stdout:(6/6): python3-jwt-1.6.1-2.el8.noarch.rpm 1.2 kB/s | 43 kB 00:34 2023-09-22T13:09:46.256 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:09:46.257 INFO:teuthology.orchestra.run.smithi069.stdout:Total 101 kB/s | 3.4 MB 00:34 2023-09-22T13:09:46.269 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:09:46.280 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:09:46.281 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:09:46.446 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:09:46.477 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:09:46.715 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:09:46.893 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-09-22T13:09:47.104 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-09-22T13:09:47.290 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-09-22T13:09:47.418 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 4/6 2023-09-22T13:09:48.012 INFO:teuthology.orchestra.run.smithi174.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 24 kB/s | 457 kB 00:19 2023-09-22T13:09:48.015 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:09:48.015 INFO:teuthology.orchestra.run.smithi174.stdout:Total 1.3 MB/s | 31 MB 00:24 2023-09-22T13:09:48.097 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:09:48.135 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:09:48.135 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:09:48.415 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 5/6 2023-09-22T13:09:48.444 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:09:48.447 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:09:48.649 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:09:48.916 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:09:48.916 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 1/6 2023-09-22T13:09:48.916 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2/6 2023-09-22T13:09:48.916 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 3/6 2023-09-22T13:09:48.916 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-09-22T13:09:48.916 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-09-22T13:09:49.509 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-09-22T13:09:49.510 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:49.510 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:09:49.510 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:09:49.510 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:09:49.510 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:09:49.510 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-22T13:09:49.510 INFO:teuthology.orchestra.run.smithi069.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-22T13:09:49.510 INFO:teuthology.orchestra.run.smithi069.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-22T13:09:49.510 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:49.510 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:09:49.534 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:09:49.631 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-22T13:09:49.743 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-22T13:09:49.958 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-22T13:09:50.189 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-09-22T13:09:50.473 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-09-22T13:09:50.481 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:03:44 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:09:50.662 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-09-22T13:09:50.836 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-09-22T13:09:51.010 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-09-22T13:09:51.213 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-09-22T13:09:51.432 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-09-22T13:09:51.671 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-09-22T13:09:51.884 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-09-22T13:09:52.062 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:09:52.064 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================================ 2023-09-22T13:09:52.064 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repository Size 2023-09-22T13:09:52.064 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================================ 2023-09-22T13:09:52.064 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:09:52.064 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 8.6 M 2023-09-22T13:09:52.065 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2023-09-22T13:09:52.065 INFO:teuthology.orchestra.run.smithi069.stdout: libgfortran x86_64 8.5.0-20.el8 baseos 645 k 2023-09-22T13:09:52.065 INFO:teuthology.orchestra.run.smithi069.stdout: libquadmath x86_64 8.5.0-20.el8 baseos 172 k 2023-09-22T13:09:52.065 INFO:teuthology.orchestra.run.smithi069.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-09-22T13:09:52.065 INFO:teuthology.orchestra.run.smithi069.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-09-22T13:09:52.065 INFO:teuthology.orchestra.run.smithi069.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-09-22T13:09:52.065 INFO:teuthology.orchestra.run.smithi069.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-09-22T13:09:52.066 INFO:teuthology.orchestra.run.smithi069.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-09-22T13:09:52.066 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:52.066 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:09:52.066 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================================ 2023-09-22T13:09:52.066 INFO:teuthology.orchestra.run.smithi069.stdout:Install 8 Packages 2023-09-22T13:09:52.066 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:09:52.067 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 37 M 2023-09-22T13:09:52.067 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 210 M 2023-09-22T13:09:52.067 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:09:52.117 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-09-22T13:09:52.523 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-09-22T13:09:53.731 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-09-22T13:09:53.952 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-09-22T13:09:54.200 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-09-22T13:09:54.409 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T13:09:54.432 INFO:teuthology.orchestra.run.smithi069.stdout:(1/8): ceph-mgr-diskprediction-local-16.2.14-5. 4.2 MB/s | 8.6 MB 00:02 2023-09-22T13:09:54.744 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-09-22T13:09:54.948 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-09-22T13:09:55.136 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-09-22T13:09:55.313 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-22T13:09:55.567 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-22T13:09:55.792 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-22T13:09:55.999 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-09-22T13:09:56.335 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-09-22T13:09:56.653 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-22T13:09:57.014 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 27/32 2023-09-22T13:09:57.187 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T13:09:58.689 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T13:09:58.865 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T13:09:59.299 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T13:09:59.473 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T13:09:59.690 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T13:09:59.882 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T13:09:59.900 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T13:10:00.006 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T13:10:00.428 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T13:10:00.429 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 1/32 2023-09-22T13:10:00.429 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2/32 2023-09-22T13:10:00.429 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 3/32 2023-09-22T13:10:00.429 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 4/32 2023-09-22T13:10:00.429 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 5/32 2023-09-22T13:10:00.429 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 6/32 2023-09-22T13:10:00.430 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-09-22T13:10:00.430 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-09-22T13:10:00.430 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-09-22T13:10:00.430 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-09-22T13:10:00.430 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-09-22T13:10:00.430 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-09-22T13:10:00.430 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-09-22T13:10:00.431 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-09-22T13:10:00.431 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-09-22T13:10:00.431 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-09-22T13:10:00.431 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T13:10:00.431 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-09-22T13:10:00.431 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-09-22T13:10:00.431 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-09-22T13:10:00.431 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-09-22T13:10:00.431 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-09-22T13:10:00.432 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-09-22T13:10:00.432 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-09-22T13:10:00.432 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-09-22T13:10:00.432 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-09-22T13:10:00.432 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-09-22T13:10:00.432 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-09-22T13:10:00.432 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-09-22T13:10:00.432 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-09-22T13:10:00.433 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-09-22T13:10:01.477 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-09-22T13:10:01.477 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:01.477 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:10:01.477 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:10:01.477 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:10:01.477 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:10:01.478 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:10:01.478 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:10:01.478 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:10:01.478 INFO:teuthology.orchestra.run.smithi174.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-22T13:10:01.478 INFO:teuthology.orchestra.run.smithi174.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-22T13:10:01.478 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-09-22T13:10:01.478 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-22T13:10:01.478 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-22T13:10:01.478 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-09-22T13:10:01.478 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-22T13:10:01.478 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-mako-1.0.6-14.el8.noarch 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-portend-2.6-1.el8.noarch 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pytz-2017.2-11.el8.noarch 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-22T13:10:01.479 INFO:teuthology.orchestra.run.smithi174.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-22T13:10:01.480 INFO:teuthology.orchestra.run.smithi174.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-22T13:10:01.480 INFO:teuthology.orchestra.run.smithi174.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-22T13:10:01.480 INFO:teuthology.orchestra.run.smithi174.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-22T13:10:01.480 INFO:teuthology.orchestra.run.smithi174.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-22T13:10:01.480 INFO:teuthology.orchestra.run.smithi174.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-22T13:10:01.480 INFO:teuthology.orchestra.run.smithi174.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-22T13:10:01.480 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:01.480 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:10:01.628 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install ceph-base 2023-09-22T13:10:02.479 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:02:22 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:10:03.867 INFO:teuthology.orchestra.run.smithi174.stdout:Package ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:10:04.047 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:10:04.048 INFO:teuthology.orchestra.run.smithi174.stdout:Nothing to do. 2023-09-22T13:10:04.048 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:10:04.147 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install cephadm 2023-09-22T13:10:05.000 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:02:25 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:10:06.562 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:10:06.563 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:10:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repository Size 2023-09-22T13:10:06.564 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:10:06.564 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:10:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 84 k 2023-09-22T13:10:06.564 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:06.564 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:10:06.565 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:10:06.565 INFO:teuthology.orchestra.run.smithi174.stdout:Install 1 Package 2023-09-22T13:10:06.565 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:06.565 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 84 k 2023-09-22T13:10:06.565 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 348 k 2023-09-22T13:10:06.565 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:10:06.725 INFO:teuthology.orchestra.run.smithi174.stdout:cephadm-16.2.14-5.g31c126b7.el8.noarch.rpm 523 kB/s | 84 kB 00:00 2023-09-22T13:10:06.726 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:10:06.726 INFO:teuthology.orchestra.run.smithi174.stdout:Total 519 kB/s | 84 kB 00:00 2023-09-22T13:10:06.727 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:10:06.733 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:10:06.733 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:10:06.735 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:10:06.738 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:10:06.781 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:10:07.241 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T13:10:07.424 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T13:10:07.654 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T13:10:08.285 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T13:10:08.285 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:08.285 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:10:08.285 INFO:teuthology.orchestra.run.smithi174.stdout: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:10:08.285 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:08.285 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:10:08.522 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install ceph-immutable-object-cache 2023-09-22T13:10:09.375 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:02:29 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:10:10.945 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:10:10.945 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:10:10.945 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repo Size 2023-09-22T13:10:10.946 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:10:10.946 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:10:10.946 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-5.g31c126b7.el8 ceph 161 k 2023-09-22T13:10:10.946 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:10.946 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:10:10.946 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:10:10.946 INFO:teuthology.orchestra.run.smithi174.stdout:Install 1 Package 2023-09-22T13:10:10.946 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:10.947 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 161 k 2023-09-22T13:10:10.947 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 470 k 2023-09-22T13:10:10.947 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:10:11.137 INFO:teuthology.orchestra.run.smithi174.stdout:ceph-immutable-object-cache-16.2.14-5.g31c126b7 846 kB/s | 161 kB 00:00 2023-09-22T13:10:11.138 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:10:11.138 INFO:teuthology.orchestra.run.smithi174.stdout:Total 841 kB/s | 161 kB 00:00 2023-09-22T13:10:11.138 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:10:11.149 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:10:11.149 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:10:11.195 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:10:11.198 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:10:11.302 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:10:11.480 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T13:10:11.886 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T13:10:12.507 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T13:10:12.507 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:12.507 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:10:12.507 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:10:12.507 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:12.507 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:10:12.709 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install ceph-mgr 2023-09-22T13:10:13.570 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:02:33 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:10:14.967 INFO:teuthology.orchestra.run.smithi174.stdout:Package ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:10:15.149 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:10:15.150 INFO:teuthology.orchestra.run.smithi174.stdout:Nothing to do. 2023-09-22T13:10:15.150 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:10:15.250 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install ceph-mgr-dashboard 2023-09-22T13:10:16.110 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:02:36 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:10:17.687 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:10:17.688 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:10:17.688 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repository Size 2023-09-22T13:10:17.689 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:10:17.689 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:10:17.689 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr-dashboard noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 3.1 M 2023-09-22T13:10:17.689 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2023-09-22T13:10:17.689 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-grafana-dashboards noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 24 k 2023-09-22T13:10:17.689 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-prometheus-alerts noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 15 k 2023-09-22T13:10:17.689 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-09-22T13:10:17.689 INFO:teuthology.orchestra.run.smithi174.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-22T13:10:17.690 INFO:teuthology.orchestra.run.smithi174.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-22T13:10:17.690 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:17.690 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:10:17.690 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:10:17.690 INFO:teuthology.orchestra.run.smithi174.stdout:Install 6 Packages 2023-09-22T13:10:17.690 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:17.691 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 3.4 M 2023-09-22T13:10:17.691 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 22 M 2023-09-22T13:10:17.691 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:10:18.465 INFO:teuthology.orchestra.run.smithi174.stdout:(1/6): ceph-grafana-dashboards-16.2.14-5.g31c12 106 kB/s | 24 kB 00:00 2023-09-22T13:10:18.490 INFO:teuthology.orchestra.run.smithi174.stdout:(2/6): ceph-prometheus-alerts-16.2.14-5.g31c126 58 kB/s | 15 kB 00:00 2023-09-22T13:10:18.590 INFO:teuthology.orchestra.run.smithi174.stdout:(3/6): ceph-mgr-dashboard-16.2.14-5.g31c126b7.e 8.8 MB/s | 3.1 MB 00:00 2023-09-22T13:10:18.807 INFO:teuthology.orchestra.run.smithi174.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 106 kB/s | 34 kB 00:00 2023-09-22T13:10:19.057 INFO:teuthology.orchestra.run.smithi174.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 421 kB/s | 196 kB 00:00 2023-09-22T13:10:30.078 INFO:teuthology.orchestra.run.smithi174.stdout:(6/6): python3-jwt-1.6.1-2.el8.noarch.rpm 3.7 kB/s | 43 kB 00:11 2023-09-22T13:10:30.081 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:10:30.081 INFO:teuthology.orchestra.run.smithi174.stdout:Total 284 kB/s | 3.4 MB 00:12 2023-09-22T13:10:30.092 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:10:30.103 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:10:30.103 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:10:30.370 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:10:30.374 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:10:30.682 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:10:30.857 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-09-22T13:10:31.069 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-09-22T13:10:31.262 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-09-22T13:10:31.382 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 4/6 2023-09-22T13:10:32.374 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 5/6 2023-09-22T13:10:32.606 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:10:32.893 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:10:32.893 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 1/6 2023-09-22T13:10:32.893 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2/6 2023-09-22T13:10:32.893 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 3/6 2023-09-22T13:10:32.893 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-09-22T13:10:32.893 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-09-22T13:10:33.522 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-09-22T13:10:33.522 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:33.523 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:10:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:10:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:10:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:10:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-22T13:10:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-22T13:10:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-22T13:10:33.523 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:33.523 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:10:33.647 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-22T13:10:34.499 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:02:54 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:10:36.072 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:10:36.072 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================================ 2023-09-22T13:10:36.073 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repository Size 2023-09-22T13:10:36.073 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================================ 2023-09-22T13:10:36.073 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:10:36.073 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 8.6 M 2023-09-22T13:10:36.073 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2023-09-22T13:10:36.073 INFO:teuthology.orchestra.run.smithi174.stdout: libgfortran x86_64 8.5.0-20.el8 baseos 645 k 2023-09-22T13:10:36.074 INFO:teuthology.orchestra.run.smithi174.stdout: libquadmath x86_64 8.5.0-20.el8 baseos 172 k 2023-09-22T13:10:36.074 INFO:teuthology.orchestra.run.smithi174.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-09-22T13:10:36.074 INFO:teuthology.orchestra.run.smithi174.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-09-22T13:10:36.074 INFO:teuthology.orchestra.run.smithi174.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-09-22T13:10:36.074 INFO:teuthology.orchestra.run.smithi174.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-09-22T13:10:36.074 INFO:teuthology.orchestra.run.smithi174.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-09-22T13:10:36.074 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:36.075 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:10:36.075 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================================ 2023-09-22T13:10:36.075 INFO:teuthology.orchestra.run.smithi174.stdout:Install 8 Packages 2023-09-22T13:10:36.075 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:10:36.076 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 37 M 2023-09-22T13:10:36.076 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 210 M 2023-09-22T13:10:36.076 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:10:38.295 INFO:teuthology.orchestra.run.smithi174.stdout:(1/8): ceph-mgr-diskprediction-local-16.2.14-5. 4.5 MB/s | 8.6 MB 00:01 2023-09-22T13:12:56.051 INFO:teuthology.orchestra.run.smithi069.stdout:(2/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 21 kB/s | 3.7 MB 03:01 2023-09-22T13:13:24.231 INFO:teuthology.orchestra.run.smithi069.stdout:(3/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 23 kB/s | 4.9 MB 03:31 2023-09-22T13:13:32.105 INFO:teuthology.orchestra.run.smithi069.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 6.2 kB/s | 225 kB 00:36 2023-09-22T13:13:38.491 INFO:teuthology.orchestra.run.smithi069.stdout:(5/8): openblas-0.3.15-4.el8.x86_64.rpm 21 kB/s | 4.7 MB 03:46 2023-09-22T13:13:39.738 INFO:teuthology.orchestra.run.smithi069.stdout:(6/8): libquadmath-8.5.0-20.el8.x86_64.rpm 127 kB/s | 172 kB 00:01 2023-09-22T13:14:01.421 INFO:teuthology.orchestra.run.smithi174.stdout:(2/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 19 kB/s | 3.7 MB 03:23 2023-09-22T13:14:08.795 INFO:teuthology.orchestra.run.smithi174.stdout:(3/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 23 kB/s | 4.9 MB 03:32 2023-09-22T13:14:16.735 INFO:teuthology.orchestra.run.smithi174.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 15 kB/s | 225 kB 00:15 2023-09-22T13:15:11.062 INFO:teuthology.orchestra.run.smithi174.stdout:(5/8): openblas-0.3.15-4.el8.x86_64.rpm 18 kB/s | 4.7 MB 04:34 2023-09-22T13:15:54.329 INFO:teuthology.orchestra.run.smithi069.stdout:[MIRROR] libgfortran-8.5.0-20.el8.x86_64.rpm: Curl error (56): Failure when receiving data from the peer for http://mirrors.unifiedlayer.com/centos/8-stream/BaseOS/x86_64/os/Packages/libgfortran-8.5.0-20.el8.x86_64.rpm [Recv failure: Connection reset by peer] 2023-09-22T13:15:54.502 INFO:teuthology.orchestra.run.smithi069.stdout:(7/8): libgfortran-8.5.0-20.el8.x86_64.rpm 4.5 kB/s | 645 kB 02:22 2023-09-22T13:16:27.886 INFO:teuthology.orchestra.run.smithi174.stdout:[MIRROR] libgfortran-8.5.0-20.el8.x86_64.rpm: Curl error (7): Couldn't connect to server for http://mirrors.unifiedlayer.com/centos/8-stream/BaseOS/x86_64/os/Packages/libgfortran-8.5.0-20.el8.x86_64.rpm [Failed to connect to mirrors.unifiedlayer.com port 80: Connection timed out] 2023-09-22T13:16:28.084 INFO:teuthology.orchestra.run.smithi174.stdout:(6/8): libgfortran-8.5.0-20.el8.x86_64.rpm 4.9 kB/s | 645 kB 02:11 2023-09-22T13:17:21.134 INFO:teuthology.orchestra.run.smithi174.stdout:[MIRROR] libquadmath-8.5.0-20.el8.x86_64.rpm: Curl error (7): Couldn't connect to server for http://mirrors.unifiedlayer.com/centos/8-stream/BaseOS/x86_64/os/Packages/libquadmath-8.5.0-20.el8.x86_64.rpm [Failed to connect to mirrors.unifiedlayer.com port 80: Connection timed out] 2023-09-22T13:17:21.285 INFO:teuthology.orchestra.run.smithi174.stdout:(7/8): libquadmath-8.5.0-20.el8.x86_64.rpm 1.3 kB/s | 172 kB 02:10 2023-09-22T13:25:02.574 INFO:teuthology.orchestra.run.smithi069.stdout:(8/8): python3-scipy-1.0.0-21.module_el8.5.0+77 20 kB/s | 14 MB 11:38 2023-09-22T13:25:02.574 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:25:02.575 INFO:teuthology.orchestra.run.smithi069.stdout:Total 41 kB/s | 37 MB 15:10 2023-09-22T13:25:02.800 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:25:02.819 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:25:02.819 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:25:03.146 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:25:03.149 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:25:03.714 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:25:03.842 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T13:25:03.985 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T13:25:04.159 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T13:25:04.754 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T13:25:04.926 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:25:05.530 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:25:05.676 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T13:25:06.362 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T13:25:06.879 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T13:25:07.117 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T13:25:08.882 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T13:25:10.161 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-09-22T13:25:10.341 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T13:25:10.637 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T13:25:10.637 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 1/8 2023-09-22T13:25:10.637 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-09-22T13:25:10.637 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:25:10.638 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-09-22T13:25:10.638 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T13:25:10.638 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-09-22T13:25:10.638 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 7/8 2023-09-22T13:25:11.278 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 8/8 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout: libgfortran-8.5.0-20.el8.x86_64 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout: libquadmath-8.5.0-20.el8.x86_64 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout: openblas-0.3.15-4.el8.x86_64 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:25:11.279 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:25:11.402 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install ceph-mgr-rook 2023-09-22T13:25:12.255 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:19:06 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:25:13.825 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:25:13.826 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:25:13.826 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repository Size 2023-09-22T13:25:13.826 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:25:13.826 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:25:13.826 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr-rook noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 27 k 2023-09-22T13:25:13.826 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2023-09-22T13:25:13.826 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-22T13:25:13.826 INFO:teuthology.orchestra.run.smithi069.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-22T13:25:13.826 INFO:teuthology.orchestra.run.smithi069.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-22T13:25:13.826 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout:Install 13 Packages 2023-09-22T13:25:13.827 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:25:13.828 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 1.9 M 2023-09-22T13:25:13.828 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 24 M 2023-09-22T13:25:13.828 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:25:14.354 INFO:teuthology.orchestra.run.smithi069.stdout:(1/13): ceph-mgr-rook-16.2.14-5.g31c126b7.el8.n 119 kB/s | 27 kB 00:00 2023-09-22T13:25:14.371 INFO:teuthology.orchestra.run.smithi069.stdout:(2/13): python3-jsonpointer-1.10-11.el8.noarch. 81 kB/s | 20 kB 00:00 2023-09-22T13:25:15.862 INFO:teuthology.orchestra.run.smithi069.stdout:(3/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 15 kB/s | 27 kB 00:01 2023-09-22T13:25:16.346 INFO:teuthology.orchestra.run.smithi069.stdout:(4/13): python3-pyasn1-modules-0.3.7-6.el8.noar 56 kB/s | 110 kB 00:01 2023-09-22T13:25:16.830 INFO:teuthology.orchestra.run.smithi069.stdout:(5/13): python3-requests-oauthlib-1.0.0-1.el8.n 88 kB/s | 43 kB 00:00 2023-09-22T13:25:17.272 INFO:teuthology.orchestra.run.smithi069.stdout:(6/13): python3-cachetools-3.1.1-4.el8.noarch.r 75 kB/s | 33 kB 00:00 2023-09-22T13:25:17.422 INFO:teuthology.orchestra.run.smithi069.stdout:(7/13): python3-certifi-2018.10.15-7.el8.noarch 109 kB/s | 16 kB 00:00 2023-09-22T13:25:17.680 INFO:teuthology.orchestra.run.smithi069.stdout:(8/13): python3-google-auth-1.1.1-10.el8.noarch 314 kB/s | 81 kB 00:00 2023-09-22T13:25:18.114 INFO:teuthology.orchestra.run.smithi069.stdout:(9/13): python3-kubernetes-11.0.0-6.el8.noarch. 2.7 MB/s | 1.2 MB 00:00 2023-09-22T13:25:18.214 INFO:teuthology.orchestra.run.smithi069.stdout:(10/13): python3-rsa-4.9-2.el8.noarch.rpm 643 kB/s | 64 kB 00:00 2023-09-22T13:25:18.323 INFO:teuthology.orchestra.run.smithi069.stdout:(11/13): python3-websocket-client-0.56.0-5.el8. 565 kB/s | 61 kB 00:00 2023-09-22T13:25:19.223 INFO:teuthology.orchestra.run.smithi069.stdout:(12/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 46 kB/s | 155 kB 00:03 2023-09-22T13:25:19.248 INFO:teuthology.orchestra.run.smithi069.stdout:(13/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 26 kB/s | 126 kB 00:04 2023-09-22T13:25:19.250 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:25:19.250 INFO:teuthology.orchestra.run.smithi069.stdout:Total 361 kB/s | 1.9 MB 00:05 2023-09-22T13:25:19.309 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:25:19.326 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:25:19.326 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:25:19.413 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:25:19.416 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:25:19.899 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:25:20.142 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-22T13:25:20.339 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-09-22T13:25:20.568 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-09-22T13:25:20.822 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-09-22T13:25:21.054 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-09-22T13:25:21.303 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-09-22T13:25:21.595 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-09-22T13:25:21.827 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-09-22T13:25:22.892 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-09-22T13:25:23.138 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-09-22T13:25:23.347 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-09-22T13:25:23.530 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-09-22T13:25:23.698 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T13:25:23.993 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T13:25:23.993 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 1/13 2023-09-22T13:25:23.993 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-09-22T13:25:23.993 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-09-22T13:25:23.993 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-09-22T13:25:23.993 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-09-22T13:25:23.993 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-22T13:25:23.993 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-22T13:25:23.993 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-09-22T13:25:23.994 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-22T13:25:23.994 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-09-22T13:25:23.994 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-09-22T13:25:23.994 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-09-22T13:25:24.655 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-09-22T13:25:24.655 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:25:24.656 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:25:24.656 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:25:24.656 INFO:teuthology.orchestra.run.smithi069.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-22T13:25:24.656 INFO:teuthology.orchestra.run.smithi069.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-22T13:25:24.656 INFO:teuthology.orchestra.run.smithi069.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-22T13:25:24.656 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-22T13:25:24.656 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-22T13:25:24.656 INFO:teuthology.orchestra.run.smithi069.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-22T13:25:24.656 INFO:teuthology.orchestra.run.smithi069.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-22T13:25:24.656 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-22T13:25:24.656 INFO:teuthology.orchestra.run.smithi069.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-22T13:25:24.657 INFO:teuthology.orchestra.run.smithi069.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-22T13:25:24.657 INFO:teuthology.orchestra.run.smithi069.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-22T13:25:24.657 INFO:teuthology.orchestra.run.smithi069.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-22T13:25:24.657 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:25:24.657 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:25:24.783 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install ceph-mgr-cephadm 2023-09-22T13:25:25.635 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:19:19 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:25:27.242 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:25:27.243 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:25:27.243 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repository Size 2023-09-22T13:25:27.243 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:25:27.243 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:25:27.243 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr-cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 116 k 2023-09-22T13:25:27.243 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2023-09-22T13:25:27.243 INFO:teuthology.orchestra.run.smithi069.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-09-22T13:25:27.244 INFO:teuthology.orchestra.run.smithi069.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-09-22T13:25:27.244 INFO:teuthology.orchestra.run.smithi069.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-09-22T13:25:27.244 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-09-22T13:25:27.244 INFO:teuthology.orchestra.run.smithi069.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-09-22T13:25:27.244 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:25:27.244 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:25:27.244 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:25:27.244 INFO:teuthology.orchestra.run.smithi069.stdout:Install 6 Packages 2023-09-22T13:25:27.244 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:25:27.245 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 5.8 M 2023-09-22T13:25:27.245 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 24 M 2023-09-22T13:25:27.245 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:25:28.007 INFO:teuthology.orchestra.run.smithi069.stdout:(1/6): ceph-mgr-cephadm-16.2.14-5.g31c126b7.el8 482 kB/s | 116 kB 00:00 2023-09-22T13:25:28.332 INFO:teuthology.orchestra.run.smithi069.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 65 kB/s | 21 kB 00:00 2023-09-22T13:25:28.633 INFO:teuthology.orchestra.run.smithi069.stdout:(3/6): python3-execnet-1.7.1-1.el8.noarch.rpm 1.2 MB/s | 355 kB 00:00 2023-09-22T13:25:28.758 INFO:teuthology.orchestra.run.smithi069.stdout:(4/6): python3-remoto-1.2.1-1.el8.noarch.rpm 391 kB/s | 49 kB 00:00 2023-09-22T13:25:58.878 INFO:teuthology.orchestra.run.smithi069.stdout:(5/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 17 kB/s | 538 kB 00:31 2023-09-22T13:27:36.274 INFO:teuthology.orchestra.run.smithi174.stdout:(8/8): python3-scipy-1.0.0-21.module_el8.5.0+77 17 kB/s | 14 MB 13:27 2023-09-22T13:27:36.274 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:27:36.275 INFO:teuthology.orchestra.run.smithi174.stdout:Total 37 kB/s | 37 MB 17:00 2023-09-22T13:27:36.505 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:27:36.523 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:27:36.524 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:27:36.850 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:27:36.853 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:27:37.406 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:27:37.558 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T13:27:37.673 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T13:27:37.808 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T13:27:38.394 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T13:27:38.592 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:27:39.192 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:27:39.341 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T13:27:40.061 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T13:27:40.412 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T13:27:40.658 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T13:27:42.426 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T13:27:43.698 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-09-22T13:27:43.915 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T13:27:44.226 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T13:27:44.226 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 1/8 2023-09-22T13:27:44.226 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-09-22T13:27:44.226 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:27:44.226 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-09-22T13:27:44.226 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T13:27:44.226 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-09-22T13:27:44.226 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 7/8 2023-09-22T13:27:44.874 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 8/8 2023-09-22T13:27:44.874 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:27:44.874 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:27:44.874 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:27:44.874 INFO:teuthology.orchestra.run.smithi174.stdout: libgfortran-8.5.0-20.el8.x86_64 2023-09-22T13:27:44.874 INFO:teuthology.orchestra.run.smithi174.stdout: libquadmath-8.5.0-20.el8.x86_64 2023-09-22T13:27:44.874 INFO:teuthology.orchestra.run.smithi174.stdout: openblas-0.3.15-4.el8.x86_64 2023-09-22T13:27:44.874 INFO:teuthology.orchestra.run.smithi174.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-09-22T13:27:44.874 INFO:teuthology.orchestra.run.smithi174.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-22T13:27:44.875 INFO:teuthology.orchestra.run.smithi174.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-22T13:27:44.875 INFO:teuthology.orchestra.run.smithi174.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-09-22T13:27:44.875 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:27:44.875 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:27:44.998 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install ceph-mgr-rook 2023-09-22T13:27:45.853 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:20:05 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:27:47.428 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:27:47.430 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:27:47.430 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repository Size 2023-09-22T13:27:47.430 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:27:47.430 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:27:47.430 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr-rook noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 27 k 2023-09-22T13:27:47.430 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2023-09-22T13:27:47.430 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-22T13:27:47.430 INFO:teuthology.orchestra.run.smithi174.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:27:47.431 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:27:47.432 INFO:teuthology.orchestra.run.smithi174.stdout:Install 13 Packages 2023-09-22T13:27:47.432 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:27:47.432 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 1.9 M 2023-09-22T13:27:47.432 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 24 M 2023-09-22T13:27:47.432 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:27:47.960 INFO:teuthology.orchestra.run.smithi174.stdout:(1/13): ceph-mgr-rook-16.2.14-5.g31c126b7.el8.n 132 kB/s | 27 kB 00:00 2023-09-22T13:27:48.368 INFO:teuthology.orchestra.run.smithi174.stdout:(2/13): python3-jsonpointer-1.10-11.el8.noarch. 32 kB/s | 20 kB 00:00 2023-09-22T13:27:48.769 INFO:teuthology.orchestra.run.smithi174.stdout:(3/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 26 kB/s | 27 kB 00:01 2023-09-22T13:27:51.594 INFO:teuthology.orchestra.run.smithi174.stdout:(4/13): python3-pyasn1-modules-0.3.7-6.el8.noar 34 kB/s | 110 kB 00:03 2023-09-22T13:27:51.919 INFO:teuthology.orchestra.run.smithi174.stdout:(5/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 32 kB/s | 126 kB 00:03 2023-09-22T13:27:52.169 INFO:teuthology.orchestra.run.smithi174.stdout:(6/13): python3-cachetools-3.1.1-4.el8.noarch.r 132 kB/s | 33 kB 00:00 2023-09-22T13:27:52.278 INFO:teuthology.orchestra.run.smithi174.stdout:(7/13): python3-certifi-2018.10.15-7.el8.noarch 150 kB/s | 16 kB 00:00 2023-09-22T13:27:52.470 INFO:teuthology.orchestra.run.smithi174.stdout:(8/13): python3-google-auth-1.1.1-10.el8.noarch 424 kB/s | 81 kB 00:00 2023-09-22T13:27:52.861 INFO:teuthology.orchestra.run.smithi174.stdout:(9/13): python3-kubernetes-11.0.0-6.el8.noarch. 3.0 MB/s | 1.2 MB 00:00 2023-09-22T13:27:52.989 INFO:teuthology.orchestra.run.smithi174.stdout:(10/13): python3-rsa-4.9-2.el8.noarch.rpm 505 kB/s | 64 kB 00:00 2023-09-22T13:27:53.237 INFO:teuthology.orchestra.run.smithi174.stdout:(11/13): python3-websocket-client-0.56.0-5.el8. 247 kB/s | 61 kB 00:00 2023-09-22T13:28:01.895 INFO:teuthology.orchestra.run.smithi174.stdout:(12/13): python3-requests-oauthlib-1.0.0-1.el8. 4.1 kB/s | 43 kB 00:10 2023-09-22T13:29:42.979 INFO:teuthology.orchestra.run.smithi069.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 19 kB/s | 4.8 MB 04:15 2023-09-22T13:29:42.980 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:29:42.981 INFO:teuthology.orchestra.run.smithi069.stdout:Total 23 kB/s | 5.8 MB 04:15 2023-09-22T13:29:43.032 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:29:43.043 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:29:43.043 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:29:43.148 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:29:43.151 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:29:43.463 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:29:43.723 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 1/6 2023-09-22T13:29:43.959 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 2/6 2023-09-22T13:29:44.904 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 3/6 2023-09-22T13:29:45.235 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 4/6 2023-09-22T13:29:45.498 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-09-22T13:29:45.690 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:29:45.973 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:29:45.973 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/6 2023-09-22T13:29:45.973 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/6 2023-09-22T13:29:45.974 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/6 2023-09-22T13:29:45.974 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 4/6 2023-09-22T13:29:45.974 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 5/6 2023-09-22T13:29:46.582 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 6/6 2023-09-22T13:29:46.609 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:29:46.610 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:29:46.610 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:29:46.610 INFO:teuthology.orchestra.run.smithi069.stdout: python3-apipkg-1.5-6.el8.noarch 2023-09-22T13:29:46.610 INFO:teuthology.orchestra.run.smithi069.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-22T13:29:46.610 INFO:teuthology.orchestra.run.smithi069.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-09-22T13:29:46.610 INFO:teuthology.orchestra.run.smithi069.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-22T13:29:46.610 INFO:teuthology.orchestra.run.smithi069.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-09-22T13:29:46.611 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:29:46.611 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:29:46.686 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install ceph-fuse 2023-09-22T13:29:47.539 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:23:41 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:29:49.115 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:29:49.116 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:29:49.116 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repository Size 2023-09-22T13:29:49.116 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:29:49.116 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:29:49.116 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 910 k 2023-09-22T13:29:49.116 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2023-09-22T13:29:49.116 INFO:teuthology.orchestra.run.smithi069.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-09-22T13:29:49.116 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:29:49.116 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:29:49.116 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:29:49.116 INFO:teuthology.orchestra.run.smithi069.stdout:Install 2 Packages 2023-09-22T13:29:49.117 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:29:49.117 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 993 k 2023-09-22T13:29:49.117 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 2.8 M 2023-09-22T13:29:49.117 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:29:49.980 INFO:teuthology.orchestra.run.smithi069.stdout:(1/2): ceph-fuse-16.2.14-5.g31c126b7.el8.x86_64 1.1 MB/s | 910 kB 00:00 2023-09-22T13:29:53.829 INFO:teuthology.orchestra.run.smithi069.stdout:(2/2): fuse-2.9.7-17.el8.x86_64.rpm 18 kB/s | 83 kB 00:04 2023-09-22T13:29:53.830 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:29:53.830 INFO:teuthology.orchestra.run.smithi069.stdout:Total 211 kB/s | 993 kB 00:04 2023-09-22T13:29:53.842 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:29:53.864 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:29:53.864 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:29:53.957 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:29:53.960 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:29:54.136 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:29:54.371 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-09-22T13:29:54.555 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:29:54.931 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:29:54.932 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/2 2023-09-22T13:29:55.539 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-09-22T13:29:55.539 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:29:55.539 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:29:55.539 INFO:teuthology.orchestra.run.smithi069.stdout: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-09-22T13:29:55.540 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:29:55.540 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:29:55.735 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install librados-devel 2023-09-22T13:29:56.595 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:23:50 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:29:58.166 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:29:58.166 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:29:58.166 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repo Size 2023-09-22T13:29:58.167 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:29:58.167 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:29:58.167 INFO:teuthology.orchestra.run.smithi069.stdout: librados-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 131 k 2023-09-22T13:29:58.167 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:29:58.167 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:29:58.167 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:29:58.167 INFO:teuthology.orchestra.run.smithi069.stdout:Install 1 Package 2023-09-22T13:29:58.167 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:29:58.168 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 131 k 2023-09-22T13:29:58.168 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 432 k 2023-09-22T13:29:58.168 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:29:58.457 INFO:teuthology.orchestra.run.smithi069.stdout:librados-devel-16.2.14-5.g31c126b7.el8.x86_64.r 452 kB/s | 131 kB 00:00 2023-09-22T13:29:58.458 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:29:58.458 INFO:teuthology.orchestra.run.smithi069.stdout:Total 450 kB/s | 131 kB 00:00 2023-09-22T13:29:58.459 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:29:58.464 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:29:58.464 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:29:58.513 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:29:58.516 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:29:58.631 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:29:58.854 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:29:58.894 INFO:teuthology.orchestra.run.smithi174.stdout:[MIRROR] python3-oauthlib-2.1.0-1.el8.noarch.rpm: Curl error (7): Couldn't connect to server for http://mirrors.unifiedlayer.com/centos/8-stream/BaseOS/x86_64/os/Packages/python3-oauthlib-2.1.0-1.el8.noarch.rpm [Failed to connect to mirrors.unifiedlayer.com port 80: Connection timed out] 2023-09-22T13:29:59.106 INFO:teuthology.orchestra.run.smithi174.stdout:(13/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 1.2 kB/s | 155 kB 02:10 2023-09-22T13:29:59.106 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:29:59.106 INFO:teuthology.orchestra.run.smithi174.stdout:Total 15 kB/s | 1.9 MB 02:11 2023-09-22T13:29:59.147 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:29:59.159 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:29:59.175 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:29:59.175 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:29:59.262 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:29:59.265 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:29:59.749 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:29:59.754 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:29:59.755 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:29:59.755 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:29:59.755 INFO:teuthology.orchestra.run.smithi069.stdout: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:29:59.755 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:29:59.756 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:29:59.944 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install libcephfs2 2023-09-22T13:29:59.998 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-22T13:30:00.186 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-09-22T13:30:00.414 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-09-22T13:30:00.643 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-09-22T13:30:00.803 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:23:54 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:30:00.867 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-09-22T13:30:01.099 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-09-22T13:30:01.391 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-09-22T13:30:01.641 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-09-22T13:30:02.181 INFO:teuthology.orchestra.run.smithi069.stdout:Package libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:30:02.361 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:30:02.362 INFO:teuthology.orchestra.run.smithi069.stdout:Nothing to do. 2023-09-22T13:30:02.362 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:30:02.553 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install libcephfs-devel 2023-09-22T13:30:02.667 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-09-22T13:30:02.959 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-09-22T13:30:03.168 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-09-22T13:30:03.360 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-09-22T13:30:03.404 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:23:57 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:30:03.527 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T13:30:03.808 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T13:30:03.808 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 1/13 2023-09-22T13:30:03.808 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-09-22T13:30:03.808 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-09-22T13:30:03.808 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-09-22T13:30:03.808 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-09-22T13:30:03.808 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-22T13:30:03.808 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-22T13:30:03.809 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-09-22T13:30:03.809 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-22T13:30:03.809 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-09-22T13:30:03.809 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-09-22T13:30:03.809 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-09-22T13:30:04.489 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-09-22T13:30:04.489 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:30:04.489 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:30:04.489 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:30:04.489 INFO:teuthology.orchestra.run.smithi174.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-22T13:30:04.490 INFO:teuthology.orchestra.run.smithi174.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-22T13:30:04.490 INFO:teuthology.orchestra.run.smithi174.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-22T13:30:04.490 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-22T13:30:04.490 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-22T13:30:04.490 INFO:teuthology.orchestra.run.smithi174.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-22T13:30:04.490 INFO:teuthology.orchestra.run.smithi174.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-22T13:30:04.490 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-22T13:30:04.490 INFO:teuthology.orchestra.run.smithi174.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-22T13:30:04.490 INFO:teuthology.orchestra.run.smithi174.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-22T13:30:04.490 INFO:teuthology.orchestra.run.smithi174.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-22T13:30:04.491 INFO:teuthology.orchestra.run.smithi174.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-22T13:30:04.491 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:30:04.491 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:30:04.607 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install ceph-mgr-cephadm 2023-09-22T13:30:04.973 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:30:04.974 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:04.974 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repo Size 2023-09-22T13:30:04.974 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:04.974 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:30:04.974 INFO:teuthology.orchestra.run.smithi069.stdout: libcephfs-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 25 k 2023-09-22T13:30:04.974 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:04.975 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:30:04.975 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:04.975 INFO:teuthology.orchestra.run.smithi069.stdout:Install 1 Package 2023-09-22T13:30:04.975 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:04.975 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 25 k 2023-09-22T13:30:04.975 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 107 k 2023-09-22T13:30:04.975 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:30:05.285 INFO:teuthology.orchestra.run.smithi069.stdout:libcephfs-devel-16.2.14-5.g31c126b7.el8.x86_64. 82 kB/s | 25 kB 00:00 2023-09-22T13:30:05.285 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:30:05.286 INFO:teuthology.orchestra.run.smithi069.stdout:Total 82 kB/s | 25 kB 00:00 2023-09-22T13:30:05.286 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:30:05.289 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:30:05.289 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:30:05.292 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:30:05.295 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:30:05.348 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:30:05.464 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:22:25 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:30:05.519 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:05.788 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:06.362 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:06.362 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:06.362 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:30:06.362 INFO:teuthology.orchestra.run.smithi069.stdout: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:30:06.362 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:06.362 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:30:06.476 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install librados2 2023-09-22T13:30:07.063 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:30:07.064 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:30:07.064 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repository Size 2023-09-22T13:30:07.064 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:30:07.064 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:30:07.064 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr-cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 116 k 2023-09-22T13:30:07.064 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2023-09-22T13:30:07.064 INFO:teuthology.orchestra.run.smithi174.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-09-22T13:30:07.064 INFO:teuthology.orchestra.run.smithi174.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-09-22T13:30:07.064 INFO:teuthology.orchestra.run.smithi174.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-09-22T13:30:07.065 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-09-22T13:30:07.065 INFO:teuthology.orchestra.run.smithi174.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-09-22T13:30:07.065 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:30:07.065 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:30:07.065 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:30:07.065 INFO:teuthology.orchestra.run.smithi174.stdout:Install 6 Packages 2023-09-22T13:30:07.065 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:30:07.066 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 5.8 M 2023-09-22T13:30:07.066 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 24 M 2023-09-22T13:30:07.066 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:30:07.325 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:24:01 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:30:08.720 INFO:teuthology.orchestra.run.smithi069.stdout:Package librados2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:30:08.900 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:30:08.901 INFO:teuthology.orchestra.run.smithi069.stdout:Nothing to do. 2023-09-22T13:30:08.901 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:30:09.055 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install librbd1 2023-09-22T13:30:09.878 INFO:teuthology.orchestra.run.smithi174.stdout:(1/6): ceph-mgr-cephadm-16.2.14-5.g31c126b7.el8 396 kB/s | 116 kB 00:00 2023-09-22T13:30:09.908 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:24:03 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:30:10.162 INFO:teuthology.orchestra.run.smithi174.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 75 kB/s | 21 kB 00:00 2023-09-22T13:30:10.387 INFO:teuthology.orchestra.run.smithi174.stdout:(3/6): python3-execnet-1.7.1-1.el8.noarch.rpm 1.5 MB/s | 355 kB 00:00 2023-09-22T13:30:10.462 INFO:teuthology.orchestra.run.smithi174.stdout:(4/6): python3-remoto-1.2.1-1.el8.noarch.rpm 653 kB/s | 49 kB 00:00 2023-09-22T13:30:11.301 INFO:teuthology.orchestra.run.smithi069.stdout:Package librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:30:11.483 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:30:11.484 INFO:teuthology.orchestra.run.smithi069.stdout:Nothing to do. 2023-09-22T13:30:11.484 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:30:11.574 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install python3-rados 2023-09-22T13:30:12.427 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:24:06 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:30:13.822 INFO:teuthology.orchestra.run.smithi069.stdout:Package python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:30:14.003 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:30:14.004 INFO:teuthology.orchestra.run.smithi069.stdout:Nothing to do. 2023-09-22T13:30:14.004 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:30:14.229 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install python3-rgw 2023-09-22T13:30:15.077 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:24:09 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:30:16.461 INFO:teuthology.orchestra.run.smithi069.stdout:Package python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:30:16.641 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:30:16.642 INFO:teuthology.orchestra.run.smithi069.stdout:Nothing to do. 2023-09-22T13:30:16.642 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:30:16.725 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install python3-cephfs 2023-09-22T13:30:17.577 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:24:11 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:30:18.966 INFO:teuthology.orchestra.run.smithi069.stdout:Package python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:30:19.150 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:30:19.192 INFO:teuthology.orchestra.run.smithi069.stdout:Nothing to do. 2023-09-22T13:30:19.192 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:30:19.350 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install python3-rbd 2023-09-22T13:30:20.202 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:24:14 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:30:21.583 INFO:teuthology.orchestra.run.smithi069.stdout:Package python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:30:21.762 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:30:21.763 INFO:teuthology.orchestra.run.smithi069.stdout:Nothing to do. 2023-09-22T13:30:21.764 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:30:21.940 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install rbd-fuse 2023-09-22T13:30:22.798 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:24:16 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:30:24.379 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:30:24.380 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:24.380 INFO:teuthology.orchestra.run.smithi069.stdout: Package Architecture Version Repository Size 2023-09-22T13:30:24.380 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:24.380 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:30:24.380 INFO:teuthology.orchestra.run.smithi069.stdout: rbd-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 88 k 2023-09-22T13:30:24.380 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:24.380 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:30:24.381 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:24.381 INFO:teuthology.orchestra.run.smithi069.stdout:Install 1 Package 2023-09-22T13:30:24.381 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:24.381 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 88 k 2023-09-22T13:30:24.381 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 240 k 2023-09-22T13:30:24.381 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:30:24.707 INFO:teuthology.orchestra.run.smithi069.stdout:rbd-fuse-16.2.14-5.g31c126b7.el8.x86_64.rpm 272 kB/s | 88 kB 00:00 2023-09-22T13:30:24.707 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:30:24.707 INFO:teuthology.orchestra.run.smithi069.stdout:Total 271 kB/s | 88 kB 00:00 2023-09-22T13:30:24.708 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:30:24.717 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:30:24.717 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:30:24.766 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:30:24.769 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:30:24.880 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:30:25.049 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:25.351 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:25.936 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:25.936 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:25.936 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:30:25.936 INFO:teuthology.orchestra.run.smithi069.stdout: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:30:25.936 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:25.937 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:30:26.151 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install rbd-mirror 2023-09-22T13:30:27.000 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:24:20 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:30:28.443 INFO:teuthology.orchestra.run.smithi174.stdout:(5/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 29 kB/s | 538 kB 00:18 2023-09-22T13:30:28.586 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:30:28.587 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:28.587 INFO:teuthology.orchestra.run.smithi069.stdout: Package Architecture Version Repository Size 2023-09-22T13:30:28.587 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:28.587 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:30:28.587 INFO:teuthology.orchestra.run.smithi069.stdout: rbd-mirror x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T13:30:28.587 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:28.587 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:30:28.587 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:28.587 INFO:teuthology.orchestra.run.smithi069.stdout:Install 1 Package 2023-09-22T13:30:28.588 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:28.588 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 3.7 M 2023-09-22T13:30:28.588 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 15 M 2023-09-22T13:30:28.588 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:30:29.049 INFO:teuthology.orchestra.run.smithi069.stdout:rbd-mirror-16.2.14-5.g31c126b7.el8.x86_64.rpm 8.0 MB/s | 3.7 MB 00:00 2023-09-22T13:30:29.050 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:30:29.050 INFO:teuthology.orchestra.run.smithi069.stdout:Total 7.9 MB/s | 3.7 MB 00:00 2023-09-22T13:30:29.050 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:30:29.061 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:30:29.062 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:30:29.125 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:30:29.128 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:30:29.582 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:30:29.792 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:30.217 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:30.811 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:30.811 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:30.811 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:30:30.811 INFO:teuthology.orchestra.run.smithi069.stdout: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:30:30.811 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:30.811 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:30:31.028 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum -y install rbd-nbd 2023-09-22T13:30:31.893 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:24:25 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:30:33.465 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:30:33.465 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:33.465 INFO:teuthology.orchestra.run.smithi069.stdout: Package Architecture Version Repository Size 2023-09-22T13:30:33.465 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:33.466 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:30:33.466 INFO:teuthology.orchestra.run.smithi069.stdout: rbd-nbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 173 k 2023-09-22T13:30:33.466 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:33.466 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:30:33.466 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:30:33.466 INFO:teuthology.orchestra.run.smithi069.stdout:Install 1 Package 2023-09-22T13:30:33.466 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:33.467 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 173 k 2023-09-22T13:30:33.467 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 515 k 2023-09-22T13:30:33.467 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:30:33.861 INFO:teuthology.orchestra.run.smithi069.stdout:rbd-nbd-16.2.14-5.g31c126b7.el8.x86_64.rpm 440 kB/s | 173 kB 00:00 2023-09-22T13:30:33.862 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:30:33.862 INFO:teuthology.orchestra.run.smithi069.stdout:Total 438 kB/s | 173 kB 00:00 2023-09-22T13:30:33.862 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:30:33.872 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:30:33.873 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:30:33.923 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:30:33.926 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:30:34.047 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:30:34.223 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:34.500 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:35.094 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:30:35.094 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:35.094 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:30:35.094 INFO:teuthology.orchestra.run.smithi069.stdout: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:30:35.095 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:30:35.095 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:30:35.252 DEBUG:teuthology.parallel:result is None 2023-09-22T13:34:52.651 INFO:teuthology.orchestra.run.smithi174.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 17 kB/s | 4.8 MB 04:43 2023-09-22T13:34:52.652 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:34:52.652 INFO:teuthology.orchestra.run.smithi174.stdout:Total 21 kB/s | 5.8 MB 04:45 2023-09-22T13:34:52.704 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:34:52.715 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:34:52.715 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:34:52.828 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:34:52.832 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:34:53.149 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:34:53.432 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 1/6 2023-09-22T13:34:53.702 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 2/6 2023-09-22T13:34:54.633 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 3/6 2023-09-22T13:34:54.987 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 4/6 2023-09-22T13:34:55.266 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-09-22T13:34:55.482 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:34:55.771 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:34:55.771 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/6 2023-09-22T13:34:55.771 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/6 2023-09-22T13:34:55.771 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/6 2023-09-22T13:34:55.772 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 4/6 2023-09-22T13:34:55.772 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 5/6 2023-09-22T13:34:56.400 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 6/6 2023-09-22T13:34:56.400 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:34:56.400 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:34:56.400 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:34:56.400 INFO:teuthology.orchestra.run.smithi174.stdout: python3-apipkg-1.5-6.el8.noarch 2023-09-22T13:34:56.401 INFO:teuthology.orchestra.run.smithi174.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-22T13:34:56.401 INFO:teuthology.orchestra.run.smithi174.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-09-22T13:34:56.401 INFO:teuthology.orchestra.run.smithi174.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-22T13:34:56.401 INFO:teuthology.orchestra.run.smithi174.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-09-22T13:34:56.401 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:34:56.401 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:34:56.500 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install ceph-fuse 2023-09-22T13:34:57.356 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:17 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:34:58.931 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:34:58.932 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:34:58.932 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repository Size 2023-09-22T13:34:58.932 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:34:58.932 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:34:58.932 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 910 k 2023-09-22T13:34:58.932 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2023-09-22T13:34:58.932 INFO:teuthology.orchestra.run.smithi174.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-09-22T13:34:58.932 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:34:58.933 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:34:58.933 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:34:58.933 INFO:teuthology.orchestra.run.smithi174.stdout:Install 2 Packages 2023-09-22T13:34:58.933 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:34:58.933 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 993 k 2023-09-22T13:34:58.934 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 2.8 M 2023-09-22T13:34:58.934 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:34:59.709 INFO:teuthology.orchestra.run.smithi174.stdout:(1/2): fuse-2.9.7-17.el8.x86_64.rpm 148 kB/s | 83 kB 00:00 2023-09-22T13:34:59.758 INFO:teuthology.orchestra.run.smithi174.stdout:(2/2): ceph-fuse-16.2.14-5.g31c126b7.el8.x86_64 1.5 MB/s | 910 kB 00:00 2023-09-22T13:34:59.758 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:34:59.759 INFO:teuthology.orchestra.run.smithi174.stdout:Total 1.2 MB/s | 993 kB 00:00 2023-09-22T13:34:59.770 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:34:59.791 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:34:59.791 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:34:59.878 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:34:59.881 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:35:00.046 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:35:00.297 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-09-22T13:35:00.507 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:35:00.907 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:35:00.907 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/2 2023-09-22T13:35:01.556 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-09-22T13:35:01.556 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:01.557 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:35:01.557 INFO:teuthology.orchestra.run.smithi174.stdout: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-09-22T13:35:01.557 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:01.557 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:01.765 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install librados-devel 2023-09-22T13:35:02.630 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:22 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:04.216 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:04.216 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:04.216 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repo Size 2023-09-22T13:35:04.216 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:04.216 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:35:04.216 INFO:teuthology.orchestra.run.smithi174.stdout: librados-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 131 k 2023-09-22T13:35:04.216 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:04.216 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:35:04.217 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:04.217 INFO:teuthology.orchestra.run.smithi174.stdout:Install 1 Package 2023-09-22T13:35:04.217 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:04.217 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 131 k 2023-09-22T13:35:04.217 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 432 k 2023-09-22T13:35:04.217 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:35:04.420 INFO:teuthology.orchestra.run.smithi174.stdout:librados-devel-16.2.14-5.g31c126b7.el8.x86_64.r 645 kB/s | 131 kB 00:00 2023-09-22T13:35:04.420 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:35:04.420 INFO:teuthology.orchestra.run.smithi174.stdout:Total 641 kB/s | 131 kB 00:00 2023-09-22T13:35:04.421 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:35:04.426 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:35:04.426 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:35:04.475 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:35:04.478 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:35:04.584 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:35:04.822 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:05.121 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:05.712 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:05.712 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:05.713 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:35:05.713 INFO:teuthology.orchestra.run.smithi174.stdout: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:35:05.713 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:05.713 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:05.894 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install libcephfs2 2023-09-22T13:35:06.758 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:26 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:08.150 INFO:teuthology.orchestra.run.smithi174.stdout:Package libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:35:08.331 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:08.332 INFO:teuthology.orchestra.run.smithi174.stdout:Nothing to do. 2023-09-22T13:35:08.332 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:08.440 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install libcephfs-devel 2023-09-22T13:35:09.296 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:29 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:10.876 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:10.877 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:10.877 INFO:teuthology.orchestra.run.smithi174.stdout: Package Arch Version Repo Size 2023-09-22T13:35:10.877 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:10.877 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:35:10.877 INFO:teuthology.orchestra.run.smithi174.stdout: libcephfs-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 25 k 2023-09-22T13:35:10.877 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:10.878 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:35:10.878 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:10.878 INFO:teuthology.orchestra.run.smithi174.stdout:Install 1 Package 2023-09-22T13:35:10.878 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:10.878 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 25 k 2023-09-22T13:35:10.878 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 107 k 2023-09-22T13:35:10.878 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:35:11.114 INFO:teuthology.orchestra.run.smithi174.stdout:libcephfs-devel-16.2.14-5.g31c126b7.el8.x86_64. 108 kB/s | 25 kB 00:00 2023-09-22T13:35:11.114 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:35:11.114 INFO:teuthology.orchestra.run.smithi174.stdout:Total 108 kB/s | 25 kB 00:00 2023-09-22T13:35:11.115 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:35:11.118 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:35:11.118 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:35:11.120 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:35:11.123 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:35:11.177 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:35:11.370 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:11.620 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:12.209 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:12.210 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:12.210 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:35:12.210 INFO:teuthology.orchestra.run.smithi174.stdout: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:35:12.210 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:12.210 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:12.334 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install librados2 2023-09-22T13:35:13.200 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:33 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:14.582 INFO:teuthology.orchestra.run.smithi174.stdout:Package librados2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:35:14.766 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:14.767 INFO:teuthology.orchestra.run.smithi174.stdout:Nothing to do. 2023-09-22T13:35:14.767 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:14.868 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install librbd1 2023-09-22T13:35:15.729 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:35 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:17.129 INFO:teuthology.orchestra.run.smithi174.stdout:Package librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:35:17.309 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:17.311 INFO:teuthology.orchestra.run.smithi174.stdout:Nothing to do. 2023-09-22T13:35:17.311 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:17.399 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install python3-rados 2023-09-22T13:35:18.268 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:38 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:19.740 INFO:teuthology.orchestra.run.smithi174.stdout:Package python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:35:19.927 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:19.928 INFO:teuthology.orchestra.run.smithi174.stdout:Nothing to do. 2023-09-22T13:35:19.928 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:20.148 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install python3-rgw 2023-09-22T13:35:21.002 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:41 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:22.391 INFO:teuthology.orchestra.run.smithi174.stdout:Package python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:35:22.572 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:22.573 INFO:teuthology.orchestra.run.smithi174.stdout:Nothing to do. 2023-09-22T13:35:22.573 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:22.655 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install python3-cephfs 2023-09-22T13:35:23.507 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:43 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:24.900 INFO:teuthology.orchestra.run.smithi174.stdout:Package python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:35:25.085 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:25.087 INFO:teuthology.orchestra.run.smithi174.stdout:Nothing to do. 2023-09-22T13:35:25.087 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:25.360 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install python3-rbd 2023-09-22T13:35:26.212 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:46 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:27.604 INFO:teuthology.orchestra.run.smithi174.stdout:Package python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:35:27.786 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:27.788 INFO:teuthology.orchestra.run.smithi174.stdout:Nothing to do. 2023-09-22T13:35:27.788 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:27.888 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install rbd-fuse 2023-09-22T13:35:28.742 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:48 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:30.306 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:30.307 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:30.307 INFO:teuthology.orchestra.run.smithi174.stdout: Package Architecture Version Repository Size 2023-09-22T13:35:30.308 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:30.308 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:35:30.308 INFO:teuthology.orchestra.run.smithi174.stdout: rbd-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 88 k 2023-09-22T13:35:30.308 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:30.308 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:35:30.308 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:30.308 INFO:teuthology.orchestra.run.smithi174.stdout:Install 1 Package 2023-09-22T13:35:30.308 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:30.308 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 88 k 2023-09-22T13:35:30.308 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 240 k 2023-09-22T13:35:30.308 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:35:30.591 INFO:teuthology.orchestra.run.smithi174.stdout:rbd-fuse-16.2.14-5.g31c126b7.el8.x86_64.rpm 312 kB/s | 88 kB 00:00 2023-09-22T13:35:30.592 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:35:30.592 INFO:teuthology.orchestra.run.smithi174.stdout:Total 311 kB/s | 88 kB 00:00 2023-09-22T13:35:30.592 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:35:30.602 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:35:30.602 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:35:30.650 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:35:30.653 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:35:30.858 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:35:31.059 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:31.347 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:31.965 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:31.965 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:31.965 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:35:31.966 INFO:teuthology.orchestra.run.smithi174.stdout: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:35:31.966 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:31.966 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:32.164 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install rbd-mirror 2023-09-22T13:35:33.021 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:53 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:34.590 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:34.708 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:34.708 INFO:teuthology.orchestra.run.smithi174.stdout: Package Architecture Version Repository Size 2023-09-22T13:35:34.708 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:34.708 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:35:34.708 INFO:teuthology.orchestra.run.smithi174.stdout: rbd-mirror x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T13:35:34.709 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:34.709 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:35:34.709 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:34.709 INFO:teuthology.orchestra.run.smithi174.stdout:Install 1 Package 2023-09-22T13:35:34.709 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:34.709 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 3.7 M 2023-09-22T13:35:34.709 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 15 M 2023-09-22T13:35:34.709 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:35:35.160 INFO:teuthology.orchestra.run.smithi174.stdout:rbd-mirror-16.2.14-5.g31c126b7.el8.x86_64.rpm 6.5 MB/s | 3.7 MB 00:00 2023-09-22T13:35:35.320 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:35:35.320 INFO:teuthology.orchestra.run.smithi174.stdout:Total 6.4 MB/s | 3.7 MB 00:00 2023-09-22T13:35:35.320 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:35:35.320 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:35:35.320 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:35:35.320 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:35:35.320 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:35:35.685 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:35:35.910 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:36.343 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:36.945 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:36.945 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:36.946 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:35:36.946 INFO:teuthology.orchestra.run.smithi174.stdout: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:35:36.946 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:36.946 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:37.186 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum -y install rbd-nbd 2023-09-22T13:35:38.050 INFO:teuthology.orchestra.run.smithi174.stdout:Last metadata expiration check: 0:27:58 ago on Fri 22 Sep 2023 01:07:40 PM UTC. 2023-09-22T13:35:39.634 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2023-09-22T13:35:39.634 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:39.634 INFO:teuthology.orchestra.run.smithi174.stdout: Package Architecture Version Repository Size 2023-09-22T13:35:39.634 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:39.634 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2023-09-22T13:35:39.634 INFO:teuthology.orchestra.run.smithi174.stdout: rbd-nbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 173 k 2023-09-22T13:35:39.634 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:39.635 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2023-09-22T13:35:39.635 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2023-09-22T13:35:39.635 INFO:teuthology.orchestra.run.smithi174.stdout:Install 1 Package 2023-09-22T13:35:39.635 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:39.635 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 173 k 2023-09-22T13:35:39.635 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 515 k 2023-09-22T13:35:39.635 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2023-09-22T13:35:39.945 INFO:teuthology.orchestra.run.smithi174.stdout:rbd-nbd-16.2.14-5.g31c126b7.el8.x86_64.rpm 557 kB/s | 173 kB 00:00 2023-09-22T13:35:39.945 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:35:39.946 INFO:teuthology.orchestra.run.smithi174.stdout:Total 555 kB/s | 173 kB 00:00 2023-09-22T13:35:39.946 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2023-09-22T13:35:39.956 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2023-09-22T13:35:39.956 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2023-09-22T13:35:40.006 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2023-09-22T13:35:40.010 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2023-09-22T13:35:40.118 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2023-09-22T13:35:40.307 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:40.581 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:41.184 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:35:41.184 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:41.184 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2023-09-22T13:35:41.184 INFO:teuthology.orchestra.run.smithi174.stdout: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:35:41.184 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:41.185 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2023-09-22T13:35:41.368 DEBUG:teuthology.parallel:result is None 2023-09-22T13:35:41.369 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:35:41.509 DEBUG:teuthology.orchestra.run.smithi069:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-22T13:35:41.537 INFO:teuthology.orchestra.run.smithi069.stdout:16.2.14-5.g31c126b7.el8 2023-09-22T13:35:41.537 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-5.g31c126b7.el8 2023-09-22T13:35:41.537 INFO:teuthology.task.install:The correct ceph version 16.2.14-5.g31c126b7 is installed. 2023-09-22T13:35:41.537 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:35:41.762 DEBUG:teuthology.orchestra.run.smithi174:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-22T13:35:41.785 INFO:teuthology.orchestra.run.smithi174.stdout:16.2.14-5.g31c126b7.el8 2023-09-22T13:35:41.786 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-5.g31c126b7.el8 2023-09-22T13:35:41.786 INFO:teuthology.task.install:The correct ceph version 16.2.14-5.g31c126b7 is installed. 2023-09-22T13:35:41.786 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-22T13:35:41.786 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:35:41.786 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-22T13:35:41.821 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:35:41.821 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-22T13:35:41.855 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-22T13:35:41.855 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:35:41.855 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/usr/bin/daemon-helper 2023-09-22T13:35:41.891 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-22T13:35:41.961 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:35:41.961 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/usr/bin/daemon-helper 2023-09-22T13:35:41.989 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-22T13:35:42.056 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-22T13:35:42.057 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:35:42.057 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-22T13:35:42.086 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-22T13:35:42.155 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:35:42.155 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-22T13:35:42.182 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-22T13:35:42.248 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-22T13:35:42.249 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:35:42.249 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/usr/bin/stdin-killer 2023-09-22T13:35:42.277 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-22T13:35:42.345 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:35:42.345 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/usr/bin/stdin-killer 2023-09-22T13:35:42.372 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-22T13:35:42.439 INFO:teuthology.run_tasks:Running task ceph... 2023-09-22T13:35:42.493 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2023-09-22T13:35:42.493 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod 777 /var/log/ceph 2023-09-22T13:35:42.495 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod 777 /var/log/ceph 2023-09-22T13:35:42.522 INFO:tasks.ceph:Disabling ceph logrotate... 2023-09-22T13:35:42.522 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -f -- /etc/logrotate.d/ceph 2023-09-22T13:35:42.562 DEBUG:teuthology.orchestra.run.smithi174:> sudo rm -f -- /etc/logrotate.d/ceph 2023-09-22T13:35:42.589 INFO:tasks.ceph:Creating extra log directories... 2023-09-22T13:35:42.590 DEBUG:teuthology.orchestra.run.smithi069:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2023-09-22T13:35:42.629 DEBUG:teuthology.orchestra.run.smithi174:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2023-09-22T13:35:42.692 INFO:tasks.ceph:Creating ceph cluster ceph... 2023-09-22T13:35:42.692 INFO:tasks.ceph:config {'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=', 'rgw lc debug interval': 10, 'rgw max chunk size': 4194304, 'rgw obj stripe size': 6291456, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'global': {'mon client directed command retry': 5, 'ms inject socket failures': 5000, 'osd heartbeat grace': 40, 'osd_max_pg_log_entries': 10, 'osd_min_pg_log_entries': 10}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon osd crush smoke test': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd fast shutdown': False, 'osd objectstore': 'bluestore', 'osd shutdown pgref assert': True}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(POOL_APP_NOT_ENABLED\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'ceph', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2023-09-22T13:35:42.692 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400937', 'branch': 'wip-cfsnyder-bucket-check-pacific', 'description': 'rgw/verify/{centos_latest clusters/fixed-2 frontend/civetweb ignore-pg-availability msgr-failures/few objectstore/bluestore-bitmap overrides proto/https rgw_pool_type/ec-profile sharding$/{default} striping$/{stripe-greater-than-chunk} tasks/{0-install bucket-check cls ragweed reshard s3tests-java s3tests versioning} validater/valgrind}', 'email': None, 'first_in_suite': False, 'job_id': '7400937', 'kernel': {'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'name': 'cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi', 'no_nested_subset': False, 'nuke-on-error': True, 'openstack': [{'volumes': {'count': 4, 'size': 10}}], 'os_type': 'centos', 'os_version': '8.stream', 'overrides': {'admin_socket': {'branch': 'wip-cfsnyder-bucket-check-pacific'}, 'ceph': {'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=', 'rgw lc debug interval': 10, 'rgw max chunk size': 4194304, 'rgw obj stripe size': 6291456, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'global': {'mon client directed command retry': 5, 'ms inject socket failures': 5000, 'osd heartbeat grace': 40, 'osd_max_pg_log_entries': 10, 'osd_min_pg_log_entries': 10}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon osd crush smoke test': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd fast shutdown': False, 'osd objectstore': 'bluestore', 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(POOL_APP_NOT_ENABLED\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {'osd default pool size': 2}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}}, 'openssl_keys': {'rgw.client.0': {'ca': 'root', 'client': 'client.0', 'embed-key': True}, 'rgw.client.1': {'ca': 'root', 'client': 'client.1', 'embed-key': True}, 'root': {'client': 'client.0', 'cn': 'teuthology', 'install': ['client.0', 'client.1'], 'key-type': 'rsa:4096'}}, 'rgw': {'client.0': {'ssl certificate': 'rgw.client.0'}, 'client.1': {'ssl certificate': 'rgw.client.1'}, 'compression type': 'random', 'ec-data-pool': True, 'erasure_code_profile': {'crush-failure-domain': 'osd', 'k': 3, 'm': 1, 'name': 'testprofile'}, 'frontend': 'civetweb', 'storage classes': 'LUKEWARM, FROZEN'}, 's3tests': {'slow_backend': True}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-cfsnyder-bucket-check-pacific', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}}, 'owner': 'scheduled_cfsnyder@teuthology', 'priority': 101, 'repo': 'https://git.ceph.com/ceph-ci.git', 'roles': [['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a'], ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'node-exporter.b']], 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'sleep_before_teardown': 0, 'suite': 'rgw', 'suite_branch': 'wip-cfsnyder-bucket-check-pacific', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph-ci.git', 'suite_sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'targets': {'smithi069.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLr7fwikwEzZnPMBsRWaZzWO3JCYp457eHBmJUE5Yu1Yo7J6l/XDwLs2hjtKMDese0SvCViCpFdttPpGjGBWMbg=', 'smithi174.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNYkpIEXE/phKHjSx0EOl4K5VhwOmLgwIRe17NNcy4gHY1AOMrfwEgRVydlL1JyjV1OzIekR1QObSJeoBChy/2U='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': None}, {'ceph': None}, {'openssl_keys': None}, {'rgw': {'client.0': None}}, {'workunit': {'clients': {'client.0': ['rgw/run-bucket-check.sh']}}}, {'workunit': {'clients': {'client.0': ['cls/test_cls_lock.sh', 'cls/test_cls_log.sh', 'cls/test_cls_refcount.sh', 'cls/test_cls_rgw.sh', 'cls/test_cls_cmpomap.sh', 'cls/test_cls_2pc_queue.sh', 'rgw/test_rgw_gc_log.sh', 'rgw/test_rgw_obj.sh', 'rgw/test_rgw_throttle.sh', 'rgw/test_librgw_file.sh']}}}, {'ragweed': {'client.0': {'default-branch': 'ceph-pacific', 'rgw_server': 'client.0', 'stages': 'prepare,check'}}}, {'workunit': {'clients': {'client.0': ['rgw/run-reshard.sh']}}}, {'s3tests-java': {'client.0': {'force-branch': 'ceph-pacific', 'force-repo': 'https://github.com/ceph/java_s3tests.git'}}}, {'s3tests': {'client.0': {'force-branch': 'ceph-pacific', 'rgw_server': 'client.0'}}}, {'workunit': {'clients': {'client.0': ['rgw/run-versioning.sh']}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '54e62bcbac4e53d9685e08328b790d3b20d71cae', 'timestamp': '2023-09-22_11:28:34', 'tube': 'smithi', 'user': 'cfsnyder', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.2830'} 2023-09-22T13:35:42.693 DEBUG:teuthology.orchestra.run.smithi069:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2023-09-22T13:35:42.698 DEBUG:teuthology.orchestra.run.smithi174:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2023-09-22T13:35:42.716 DEBUG:teuthology.orchestra.run.smithi069:> sudo install -d -m0777 -- /var/run/ceph 2023-09-22T13:35:42.756 DEBUG:teuthology.orchestra.run.smithi174:> sudo install -d -m0777 -- /var/run/ceph 2023-09-22T13:35:42.783 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:35:42.783 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/scratch_devs of=/dev/stdout 2023-09-22T13:35:42.838 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-22T13:35:42.839 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_1 2023-09-22T13:35:42.896 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-22T13:35:42.896 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:35:42.896 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 6h/6d Inode: 417273 Links: 1 2023-09-22T13:35:42.896 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:35:42.896 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:35:42.897 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2023-09-22 13:30:29.968599109 +0000 2023-09-22T13:35:42.897 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2023-09-22 12:32:11.653618454 +0000 2023-09-22T13:35:42.897 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2023-09-22 12:32:11.653618454 +0000 2023-09-22T13:35:42.897 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: - 2023-09-22T13:35:42.897 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-22T13:35:42.961 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2023-09-22T13:35:42.961 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2023-09-22T13:35:42.962 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.00022542 s, 2.3 MB/s 2023-09-22T13:35:42.962 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-22T13:35:43.021 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_2 2023-09-22T13:35:43.079 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-22T13:35:43.079 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:35:43.079 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 6h/6d Inode: 424009 Links: 1 2023-09-22T13:35:43.079 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:35:43.080 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:35:43.080 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2023-09-22 13:30:29.968599109 +0000 2023-09-22T13:35:43.080 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2023-09-22 12:32:12.143607810 +0000 2023-09-22T13:35:43.080 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2023-09-22 12:32:12.143607810 +0000 2023-09-22T13:35:43.080 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: - 2023-09-22T13:35:43.080 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-22T13:35:43.145 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2023-09-22T13:35:43.145 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2023-09-22T13:35:43.145 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000199918 s, 2.6 MB/s 2023-09-22T13:35:43.146 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-22T13:35:43.205 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_3 2023-09-22T13:35:43.263 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-22T13:35:43.264 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:35:43.264 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 6h/6d Inode: 425994 Links: 1 2023-09-22T13:35:43.264 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:35:43.264 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:35:43.264 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2023-09-22 13:30:29.968599109 +0000 2023-09-22T13:35:43.264 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2023-09-22 12:32:12.636597102 +0000 2023-09-22T13:35:43.264 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2023-09-22 12:32:12.636597102 +0000 2023-09-22T13:35:43.264 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: - 2023-09-22T13:35:43.264 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-22T13:35:43.329 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2023-09-22T13:35:43.329 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2023-09-22T13:35:43.329 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000237664 s, 2.2 MB/s 2023-09-22T13:35:43.330 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-22T13:35:43.389 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_4 2023-09-22T13:35:43.447 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-22T13:35:43.447 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:35:43.448 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 6h/6d Inode: 426102 Links: 1 2023-09-22T13:35:43.448 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:35:43.448 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:35:43.448 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2023-09-22 13:30:29.968599109 +0000 2023-09-22T13:35:43.448 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2023-09-22 12:32:13.138586198 +0000 2023-09-22T13:35:43.448 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2023-09-22 12:32:13.138586198 +0000 2023-09-22T13:35:43.448 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: - 2023-09-22T13:35:43.448 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-22T13:35:43.513 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2023-09-22T13:35:43.513 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2023-09-22T13:35:43.513 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000239901 s, 2.1 MB/s 2023-09-22T13:35:43.514 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-22T13:35:43.573 INFO:tasks.ceph:osd dev map: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:35:43.573 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:35:43.573 DEBUG:teuthology.orchestra.run.smithi174:> dd if=/scratch_devs of=/dev/stdout 2023-09-22T13:35:43.593 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-22T13:35:43.594 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_1 2023-09-22T13:35:43.654 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-22T13:35:43.654 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:35:43.654 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 6h/6d Inode: 424993 Links: 1 2023-09-22T13:35:43.654 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:35:43.654 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:35:43.654 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2023-09-22 13:35:36.087751036 +0000 2023-09-22T13:35:43.654 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2023-09-22 12:31:58.261892964 +0000 2023-09-22T13:35:43.654 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2023-09-22 12:31:58.261892964 +0000 2023-09-22T13:35:43.654 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: - 2023-09-22T13:35:43.655 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-22T13:35:43.723 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2023-09-22T13:35:43.723 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2023-09-22T13:35:43.723 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000255784 s, 2.0 MB/s 2023-09-22T13:35:43.725 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-22T13:35:43.786 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_2 2023-09-22T13:35:43.845 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-22T13:35:43.845 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:35:43.845 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 6h/6d Inode: 423192 Links: 1 2023-09-22T13:35:43.845 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:35:43.845 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:35:43.845 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2023-09-22 13:35:36.087751036 +0000 2023-09-22T13:35:43.845 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2023-09-22 12:31:58.756886698 +0000 2023-09-22T13:35:43.845 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2023-09-22 12:31:58.756886698 +0000 2023-09-22T13:35:43.846 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: - 2023-09-22T13:35:43.846 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-22T13:35:43.912 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2023-09-22T13:35:43.913 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2023-09-22T13:35:43.913 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000205101 s, 2.5 MB/s 2023-09-22T13:35:43.914 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-22T13:35:43.974 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_3 2023-09-22T13:35:44.034 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-22T13:35:44.034 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:35:44.034 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 6h/6d Inode: 426103 Links: 1 2023-09-22T13:35:44.034 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:35:44.034 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:35:44.034 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2023-09-22 13:35:36.088751023 +0000 2023-09-22T13:35:44.035 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2023-09-22 12:31:59.240880571 +0000 2023-09-22T13:35:44.035 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2023-09-22 12:31:59.240880571 +0000 2023-09-22T13:35:44.035 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: - 2023-09-22T13:35:44.035 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-22T13:35:44.100 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2023-09-22T13:35:44.100 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2023-09-22T13:35:44.100 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000212034 s, 2.4 MB/s 2023-09-22T13:35:44.101 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-22T13:35:44.160 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_4 2023-09-22T13:35:44.219 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-22T13:35:44.219 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:35:44.219 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 6h/6d Inode: 423311 Links: 1 2023-09-22T13:35:44.219 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:35:44.219 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:35:44.219 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2023-09-22 13:35:36.088751023 +0000 2023-09-22T13:35:44.219 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2023-09-22 12:31:59.742874217 +0000 2023-09-22T13:35:44.219 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2023-09-22 12:31:59.742874217 +0000 2023-09-22T13:35:44.219 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: - 2023-09-22T13:35:44.220 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-22T13:35:44.285 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2023-09-22T13:35:44.285 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2023-09-22T13:35:44.285 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000249329 s, 2.1 MB/s 2023-09-22T13:35:44.286 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-22T13:35:44.344 INFO:tasks.ceph:osd dev map: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:35:44.344 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi069.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi174.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2023-09-22T13:35:44.345 INFO:tasks.ceph:Generating config... 2023-09-22T13:35:44.346 INFO:tasks.ceph:[client] debug rgw = 20 2023-09-22T13:35:44.346 INFO:tasks.ceph:[client] rgw crypt require ssl = False 2023-09-22T13:35:44.346 INFO:tasks.ceph:[client] rgw crypt s3 kms backend = testing 2023-09-22T13:35:44.346 INFO:tasks.ceph:[client] rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= 2023-09-22T13:35:44.346 INFO:tasks.ceph:[client] rgw lc debug interval = 10 2023-09-22T13:35:44.346 INFO:tasks.ceph:[client] rgw max chunk size = 4194304 2023-09-22T13:35:44.346 INFO:tasks.ceph:[client] rgw obj stripe size = 6291456 2023-09-22T13:35:44.346 INFO:tasks.ceph:[client] setgroup = ceph 2023-09-22T13:35:44.346 INFO:tasks.ceph:[client] setuser = ceph 2023-09-22T13:35:44.346 INFO:tasks.ceph:[global] mon client directed command retry = 5 2023-09-22T13:35:44.346 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2023-09-22T13:35:44.347 INFO:tasks.ceph:[global] osd heartbeat grace = 40 2023-09-22T13:35:44.347 INFO:tasks.ceph:[global] osd_max_pg_log_entries = 10 2023-09-22T13:35:44.347 INFO:tasks.ceph:[global] osd_min_pg_log_entries = 10 2023-09-22T13:35:44.347 INFO:tasks.ceph:[mgr] debug mgr = 20 2023-09-22T13:35:44.347 INFO:tasks.ceph:[mgr] debug ms = 1 2023-09-22T13:35:44.347 INFO:tasks.ceph:[mon] debug mon = 20 2023-09-22T13:35:44.347 INFO:tasks.ceph:[mon] debug ms = 1 2023-09-22T13:35:44.347 INFO:tasks.ceph:[mon] debug paxos = 20 2023-09-22T13:35:44.347 INFO:tasks.ceph:[mon] mon osd crush smoke test = False 2023-09-22T13:35:44.347 INFO:tasks.ceph:[osd] bdev async discard = True 2023-09-22T13:35:44.347 INFO:tasks.ceph:[osd] bdev enable discard = True 2023-09-22T13:35:44.347 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2023-09-22T13:35:44.347 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] debug ms = 1 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] debug osd = 20 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] osd fast shutdown = False 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2023-09-22T13:35:44.348 INFO:tasks.ceph:[osd] osd shutdown pgref assert = True 2023-09-22T13:35:44.349 INFO:tasks.ceph:Setting up mon.a... 2023-09-22T13:35:44.349 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring 2023-09-22T13:35:44.468 INFO:teuthology.orchestra.run.smithi069.stdout:creating /etc/ceph/ceph.keyring 2023-09-22T13:35:44.470 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring 2023-09-22T13:35:44.514 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod 0644 /etc/ceph/ceph.keyring 2023-09-22T13:35:44.582 DEBUG:tasks.ceph:Ceph mon addresses: [('mon.a', '172.21.15.69'), ('mon.c', '[v2:172.21.15.69:3301,v1:172.21.15.69:6790]'), ('mon.b', '172.21.15.174')] 2023-09-22T13:35:44.582 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.69,[v2:172.21.15.69:3301,v1:172.21.15.69:6790],172.21.15.174', 'mon client directed command retry': 5, 'ms inject socket failures': 5000, 'osd heartbeat grace': 40, 'osd_max_pg_log_entries': 10, 'osd_min_pg_log_entries': 10}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd fast shutdown': False, 'osd objectstore': 'bluestore', 'osd shutdown pgref assert': True}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon osd crush smoke test': False}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=', 'rgw lc debug interval': 10, 'rgw max chunk size': 4194304, 'rgw obj stripe size': 6291456, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mon.a': {}, 'mon.c': {}, 'mon.b': {}} 2023-09-22T13:35:44.583 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:35:44.583 DEBUG:teuthology.orchestra.run.smithi069:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2023-09-22T13:35:44.640 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.69 --addv c '[v2:172.21.15.69:3301,v1:172.21.15.69:6790]' --add b 172.21.15.174 --print /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:35:44.725 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:35:44.725 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:35:44.725 INFO:teuthology.orchestra.run.smithi069.stdout:monmaptool: monmap file /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:35:44.726 INFO:teuthology.orchestra.run.smithi069.stdout:monmaptool: generated fsid 328c464b-5dfe-4280-b09b-dba594b52bd4 2023-09-22T13:35:44.726 INFO:teuthology.orchestra.run.smithi069.stdout:epoch 0 2023-09-22T13:35:44.726 INFO:teuthology.orchestra.run.smithi069.stdout:fsid 328c464b-5dfe-4280-b09b-dba594b52bd4 2023-09-22T13:35:44.726 INFO:teuthology.orchestra.run.smithi069.stdout:last_changed 2023-09-22T13:35:44.726067+0000 2023-09-22T13:35:44.726 INFO:teuthology.orchestra.run.smithi069.stdout:created 2023-09-22T13:35:44.726067+0000 2023-09-22T13:35:44.726 INFO:teuthology.orchestra.run.smithi069.stdout:min_mon_release 0 (unknown) 2023-09-22T13:35:44.726 INFO:teuthology.orchestra.run.smithi069.stdout:election_strategy: 1 2023-09-22T13:35:44.726 INFO:teuthology.orchestra.run.smithi069.stdout:0: [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon.a 2023-09-22T13:35:44.726 INFO:teuthology.orchestra.run.smithi069.stdout:1: [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon.b 2023-09-22T13:35:44.726 INFO:teuthology.orchestra.run.smithi069.stdout:2: [v2:172.21.15.69:3301/0,v1:172.21.15.69:6790/0] mon.c 2023-09-22T13:35:44.727 INFO:teuthology.orchestra.run.smithi069.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/ceph.monmap (3 monitors) 2023-09-22T13:35:44.728 DEBUG:teuthology.orchestra.run.smithi069:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2023-09-22T13:35:44.745 INFO:tasks.ceph:Writing /etc/ceph/ceph.conf for FSID 328c464b-5dfe-4280-b09b-dba594b52bd4... 2023-09-22T13:35:44.746 DEBUG:teuthology.orchestra.run.smithi069:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2023-09-22T13:35:44.787 DEBUG:teuthology.orchestra.run.smithi174:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2023-09-22T13:35:44.830 INFO:teuthology.orchestra.run.smithi069.stdout:[global] 2023-09-22T13:35:44.830 INFO:teuthology.orchestra.run.smithi069.stdout: chdir = "" 2023-09-22T13:35:44.830 INFO:teuthology.orchestra.run.smithi069.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2023-09-22T13:35:44.831 INFO:teuthology.orchestra.run.smithi069.stdout: auth supported = cephx 2023-09-22T13:35:44.831 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.831 INFO:teuthology.orchestra.run.smithi069.stdout: filestore xattr use omap = true 2023-09-22T13:35:44.831 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.832 INFO:teuthology.orchestra.run.smithi069.stdout: mon clock drift allowed = 1.000 2023-09-22T13:35:44.832 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.832 INFO:teuthology.orchestra.run.smithi069.stdout: osd crush chooseleaf type = 0 2023-09-22T13:35:44.832 INFO:teuthology.orchestra.run.smithi069.stdout: auth debug = true 2023-09-22T13:35:44.832 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.832 INFO:teuthology.orchestra.run.smithi069.stdout: ms die on old message = true 2023-09-22T13:35:44.832 INFO:teuthology.orchestra.run.smithi069.stdout: ms die on bug = true 2023-09-22T13:35:44.833 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.833 INFO:teuthology.orchestra.run.smithi069.stdout: mon max pg per osd = 10000 # >= luminous 2023-09-22T13:35:44.833 INFO:teuthology.orchestra.run.smithi069.stdout: mon pg warn max object skew = 0 2023-09-22T13:35:44.833 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.833 INFO:teuthology.orchestra.run.smithi069.stdout: # disable pg_autoscaler by default for new pools 2023-09-22T13:35:44.833 INFO:teuthology.orchestra.run.smithi069.stdout: osd_pool_default_pg_autoscale_mode = off 2023-09-22T13:35:44.833 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.833 INFO:teuthology.orchestra.run.smithi069.stdout: osd pool default size = 2 2023-09-22T13:35:44.833 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.833 INFO:teuthology.orchestra.run.smithi069.stdout: mon osd allow primary affinity = true 2023-09-22T13:35:44.833 INFO:teuthology.orchestra.run.smithi069.stdout: mon osd allow pg remap = true 2023-09-22T13:35:44.834 INFO:teuthology.orchestra.run.smithi069.stdout: mon warn on legacy crush tunables = false 2023-09-22T13:35:44.834 INFO:teuthology.orchestra.run.smithi069.stdout: mon warn on crush straw calc version zero = false 2023-09-22T13:35:44.834 INFO:teuthology.orchestra.run.smithi069.stdout: mon warn on no sortbitwise = false 2023-09-22T13:35:44.834 INFO:teuthology.orchestra.run.smithi069.stdout: mon warn on osd down out interval zero = false 2023-09-22T13:35:44.834 INFO:teuthology.orchestra.run.smithi069.stdout: mon warn on too few osds = false 2023-09-22T13:35:44.834 INFO:teuthology.orchestra.run.smithi069.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2023-09-22T13:35:44.834 INFO:teuthology.orchestra.run.smithi069.stdout: mon_warn_on_pool_no_redundancy = false 2023-09-22T13:35:44.834 INFO:teuthology.orchestra.run.smithi069.stdout: mon_allow_pool_size_one = true 2023-09-22T13:35:44.834 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.834 INFO:teuthology.orchestra.run.smithi069.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: osd default data pool replay window = 5 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: mon allow pool delete = true 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: mon cluster log file level = debug 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: debug asserts on shutdown = true 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: mon health detail to clog = false 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: mon host = "172.21.15.69,[v2:172.21.15.69:3301,v1:172.21.15.69:6790],172.21.15.174" 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: mon client directed command retry = 5 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: ms inject socket failures = 5000 2023-09-22T13:35:44.835 INFO:teuthology.orchestra.run.smithi069.stdout: osd heartbeat grace = 40 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout: osd_max_pg_log_entries = 10 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout: osd_min_pg_log_entries = 10 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout: fsid = 328c464b-5dfe-4280-b09b-dba594b52bd4 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout:[osd] 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout: osd journal size = 100 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout: osd scrub load threshold = 5.0 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout: osd scrub max interval = 600 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout: osd recover clone overlap = true 2023-09-22T13:35:44.836 INFO:teuthology.orchestra.run.smithi069.stdout: osd recovery max chunk = 1048576 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: osd debug shutdown = true 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: osd debug op order = true 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: osd debug verify stray on activate = true 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: osd open classes on start = true 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: osd debug pg log writeout = true 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: osd deep scrub update digest min age = 30 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: osd map max advance = 10 2023-09-22T13:35:44.837 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.838 INFO:teuthology.orchestra.run.smithi069.stdout: journal zero on create = true 2023-09-22T13:35:44.838 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.838 INFO:teuthology.orchestra.run.smithi069.stdout: filestore ondisk finisher threads = 3 2023-09-22T13:35:44.838 INFO:teuthology.orchestra.run.smithi069.stdout: filestore apply finisher threads = 3 2023-09-22T13:35:44.838 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.838 INFO:teuthology.orchestra.run.smithi069.stdout: bdev debug aio = true 2023-09-22T13:35:44.838 INFO:teuthology.orchestra.run.smithi069.stdout: osd debug misdirected ops = true 2023-09-22T13:35:44.838 INFO:teuthology.orchestra.run.smithi069.stdout: bdev async discard = True 2023-09-22T13:35:44.838 INFO:teuthology.orchestra.run.smithi069.stdout: bdev enable discard = True 2023-09-22T13:35:44.838 INFO:teuthology.orchestra.run.smithi069.stdout: bluestore allocator = bitmap 2023-09-22T13:35:44.838 INFO:teuthology.orchestra.run.smithi069.stdout: bluestore block size = 96636764160 2023-09-22T13:35:44.839 INFO:teuthology.orchestra.run.smithi069.stdout: bluestore fsck on mount = True 2023-09-22T13:35:44.839 INFO:teuthology.orchestra.run.smithi069.stdout: debug bluefs = 1/20 2023-09-22T13:35:44.839 INFO:teuthology.orchestra.run.smithi069.stdout: debug bluestore = 1/20 2023-09-22T13:35:44.839 INFO:teuthology.orchestra.run.smithi069.stdout: debug ms = 1 2023-09-22T13:35:44.839 INFO:teuthology.orchestra.run.smithi069.stdout: debug osd = 20 2023-09-22T13:35:44.839 INFO:teuthology.orchestra.run.smithi069.stdout: debug rocksdb = 4/10 2023-09-22T13:35:44.839 INFO:teuthology.orchestra.run.smithi069.stdout: mon osd backfillfull_ratio = 0.85 2023-09-22T13:35:44.839 INFO:teuthology.orchestra.run.smithi069.stdout: mon osd full ratio = 0.9 2023-09-22T13:35:44.839 INFO:teuthology.orchestra.run.smithi069.stdout: mon osd nearfull ratio = 0.8 2023-09-22T13:35:44.839 INFO:teuthology.orchestra.run.smithi069.stdout: osd failsafe full ratio = 0.95 2023-09-22T13:35:44.839 INFO:teuthology.orchestra.run.smithi069.stdout: osd fast shutdown = False 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout: osd objectstore = bluestore 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout: osd shutdown pgref assert = True 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout:[mgr] 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout: debug ms = 1 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout: debug mgr = 20 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout: debug mon = 20 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout: debug auth = 20 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout: mgr/telemetry/nag = false 2023-09-22T13:35:44.840 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout:[mon] 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout: debug ms = 1 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout: debug mon = 20 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout: debug paxos = 20 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout: debug auth = 20 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout: mon data avail warn = 5 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout: mon mgr mkfs grace = 240 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout: mon osd reporter subtree level = osd 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout: mon osd prime pg temp = true 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:35:44.841 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.842 INFO:teuthology.orchestra.run.smithi069.stdout: # rotate auth tickets quickly to exercise renewal paths 2023-09-22T13:35:44.842 INFO:teuthology.orchestra.run.smithi069.stdout: auth mon ticket ttl = 660 # 11m 2023-09-22T13:35:44.842 INFO:teuthology.orchestra.run.smithi069.stdout: auth service ticket ttl = 240 # 4m 2023-09-22T13:35:44.842 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.842 INFO:teuthology.orchestra.run.smithi069.stdout: # don't complain about insecure global_id in the test suite 2023-09-22T13:35:44.842 INFO:teuthology.orchestra.run.smithi069.stdout: mon_warn_on_insecure_global_id_reclaim = false 2023-09-22T13:35:44.842 INFO:teuthology.orchestra.run.smithi069.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2023-09-22T13:35:44.842 INFO:teuthology.orchestra.run.smithi069.stdout: mon osd crush smoke test = False 2023-09-22T13:35:44.842 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:35:44.842 INFO:teuthology.orchestra.run.smithi069.stdout:[client] 2023-09-22T13:35:44.842 INFO:teuthology.orchestra.run.smithi069.stdout: rgw cache enabled = true 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: rgw enable ops log = true 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: rgw enable usage log = true 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: debug rgw = 20 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: rgw crypt require ssl = False 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: rgw crypt s3 kms backend = testing 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: rgw lc debug interval = 10 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: rgw max chunk size = 4194304 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: rgw obj stripe size = 6291456 2023-09-22T13:35:44.843 INFO:teuthology.orchestra.run.smithi069.stdout: setgroup = ceph 2023-09-22T13:35:44.844 INFO:teuthology.orchestra.run.smithi069.stdout: setuser = ceph 2023-09-22T13:35:44.844 INFO:teuthology.orchestra.run.smithi069.stdout:[mon.a] 2023-09-22T13:35:44.844 INFO:teuthology.orchestra.run.smithi069.stdout:[mon.c] 2023-09-22T13:35:44.844 INFO:teuthology.orchestra.run.smithi069.stdout:[mon.b] 2023-09-22T13:35:44.844 INFO:teuthology.orchestra.run.smithi174.stdout:[global] 2023-09-22T13:35:44.844 INFO:teuthology.orchestra.run.smithi174.stdout: chdir = "" 2023-09-22T13:35:44.845 INFO:teuthology.orchestra.run.smithi174.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2023-09-22T13:35:44.845 INFO:teuthology.orchestra.run.smithi174.stdout: auth supported = cephx 2023-09-22T13:35:44.845 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.845 INFO:teuthology.orchestra.run.smithi174.stdout: filestore xattr use omap = true 2023-09-22T13:35:44.845 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.845 INFO:teuthology.orchestra.run.smithi174.stdout: mon clock drift allowed = 1.000 2023-09-22T13:35:44.845 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.845 INFO:teuthology.orchestra.run.smithi174.stdout: osd crush chooseleaf type = 0 2023-09-22T13:35:44.845 INFO:teuthology.orchestra.run.smithi174.stdout: auth debug = true 2023-09-22T13:35:44.845 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.845 INFO:teuthology.orchestra.run.smithi174.stdout: ms die on old message = true 2023-09-22T13:35:44.846 INFO:teuthology.orchestra.run.smithi174.stdout: ms die on bug = true 2023-09-22T13:35:44.846 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.846 INFO:teuthology.orchestra.run.smithi174.stdout: mon max pg per osd = 10000 # >= luminous 2023-09-22T13:35:44.846 INFO:teuthology.orchestra.run.smithi174.stdout: mon pg warn max object skew = 0 2023-09-22T13:35:44.846 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.846 INFO:teuthology.orchestra.run.smithi174.stdout: # disable pg_autoscaler by default for new pools 2023-09-22T13:35:44.846 INFO:teuthology.orchestra.run.smithi174.stdout: osd_pool_default_pg_autoscale_mode = off 2023-09-22T13:35:44.846 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.846 INFO:teuthology.orchestra.run.smithi174.stdout: osd pool default size = 2 2023-09-22T13:35:44.846 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.846 INFO:teuthology.orchestra.run.smithi174.stdout: mon osd allow primary affinity = true 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: mon osd allow pg remap = true 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: mon warn on legacy crush tunables = false 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: mon warn on crush straw calc version zero = false 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: mon warn on no sortbitwise = false 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: mon warn on osd down out interval zero = false 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: mon warn on too few osds = false 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: mon_warn_on_pool_no_redundancy = false 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: mon_allow_pool_size_one = true 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd 2023-09-22T13:35:44.847 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.848 INFO:teuthology.orchestra.run.smithi174.stdout: osd default data pool replay window = 5 2023-09-22T13:35:44.848 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.848 INFO:teuthology.orchestra.run.smithi174.stdout: mon allow pool delete = true 2023-09-22T13:35:44.848 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.848 INFO:teuthology.orchestra.run.smithi174.stdout: mon cluster log file level = debug 2023-09-22T13:35:44.848 INFO:teuthology.orchestra.run.smithi174.stdout: debug asserts on shutdown = true 2023-09-22T13:35:44.848 INFO:teuthology.orchestra.run.smithi174.stdout: mon health detail to clog = false 2023-09-22T13:35:44.848 INFO:teuthology.orchestra.run.smithi174.stdout: mon host = "172.21.15.69,[v2:172.21.15.69:3301,v1:172.21.15.69:6790],172.21.15.174" 2023-09-22T13:35:44.848 INFO:teuthology.orchestra.run.smithi174.stdout: mon client directed command retry = 5 2023-09-22T13:35:44.848 INFO:teuthology.orchestra.run.smithi174.stdout: ms inject socket failures = 5000 2023-09-22T13:35:44.848 INFO:teuthology.orchestra.run.smithi174.stdout: osd heartbeat grace = 40 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout: osd_max_pg_log_entries = 10 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout: osd_min_pg_log_entries = 10 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout: fsid = 328c464b-5dfe-4280-b09b-dba594b52bd4 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout:[osd] 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout: osd journal size = 100 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout: osd scrub load threshold = 5.0 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout: osd scrub max interval = 600 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout: osd recover clone overlap = true 2023-09-22T13:35:44.849 INFO:teuthology.orchestra.run.smithi174.stdout: osd recovery max chunk = 1048576 2023-09-22T13:35:44.850 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.850 INFO:teuthology.orchestra.run.smithi174.stdout: osd debug shutdown = true 2023-09-22T13:35:44.850 INFO:teuthology.orchestra.run.smithi174.stdout: osd debug op order = true 2023-09-22T13:35:44.850 INFO:teuthology.orchestra.run.smithi174.stdout: osd debug verify stray on activate = true 2023-09-22T13:35:44.850 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.850 INFO:teuthology.orchestra.run.smithi174.stdout: osd open classes on start = true 2023-09-22T13:35:44.850 INFO:teuthology.orchestra.run.smithi174.stdout: osd debug pg log writeout = true 2023-09-22T13:35:44.850 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.850 INFO:teuthology.orchestra.run.smithi174.stdout: osd deep scrub update digest min age = 30 2023-09-22T13:35:44.850 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.850 INFO:teuthology.orchestra.run.smithi174.stdout: osd map max advance = 10 2023-09-22T13:35:44.851 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.851 INFO:teuthology.orchestra.run.smithi174.stdout: journal zero on create = true 2023-09-22T13:35:44.851 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.851 INFO:teuthology.orchestra.run.smithi174.stdout: filestore ondisk finisher threads = 3 2023-09-22T13:35:44.851 INFO:teuthology.orchestra.run.smithi174.stdout: filestore apply finisher threads = 3 2023-09-22T13:35:44.851 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.851 INFO:teuthology.orchestra.run.smithi174.stdout: bdev debug aio = true 2023-09-22T13:35:44.851 INFO:teuthology.orchestra.run.smithi174.stdout: osd debug misdirected ops = true 2023-09-22T13:35:44.851 INFO:teuthology.orchestra.run.smithi174.stdout: bdev async discard = True 2023-09-22T13:35:44.851 INFO:teuthology.orchestra.run.smithi174.stdout: bdev enable discard = True 2023-09-22T13:35:44.852 INFO:teuthology.orchestra.run.smithi174.stdout: bluestore allocator = bitmap 2023-09-22T13:35:44.852 INFO:teuthology.orchestra.run.smithi174.stdout: bluestore block size = 96636764160 2023-09-22T13:35:44.852 INFO:teuthology.orchestra.run.smithi174.stdout: bluestore fsck on mount = True 2023-09-22T13:35:44.852 INFO:teuthology.orchestra.run.smithi174.stdout: debug bluefs = 1/20 2023-09-22T13:35:44.852 INFO:teuthology.orchestra.run.smithi174.stdout: debug bluestore = 1/20 2023-09-22T13:35:44.852 INFO:teuthology.orchestra.run.smithi174.stdout: debug ms = 1 2023-09-22T13:35:44.852 INFO:teuthology.orchestra.run.smithi174.stdout: debug osd = 20 2023-09-22T13:35:44.852 INFO:teuthology.orchestra.run.smithi174.stdout: debug rocksdb = 4/10 2023-09-22T13:35:44.852 INFO:teuthology.orchestra.run.smithi174.stdout: mon osd backfillfull_ratio = 0.85 2023-09-22T13:35:44.852 INFO:teuthology.orchestra.run.smithi174.stdout: mon osd full ratio = 0.9 2023-09-22T13:35:44.852 INFO:teuthology.orchestra.run.smithi174.stdout: mon osd nearfull ratio = 0.8 2023-09-22T13:35:44.853 INFO:teuthology.orchestra.run.smithi174.stdout: osd failsafe full ratio = 0.95 2023-09-22T13:35:44.853 INFO:teuthology.orchestra.run.smithi174.stdout: osd fast shutdown = False 2023-09-22T13:35:44.853 INFO:teuthology.orchestra.run.smithi174.stdout: osd objectstore = bluestore 2023-09-22T13:35:44.853 INFO:teuthology.orchestra.run.smithi174.stdout: osd shutdown pgref assert = True 2023-09-22T13:35:44.853 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.853 INFO:teuthology.orchestra.run.smithi174.stdout:[mgr] 2023-09-22T13:35:44.853 INFO:teuthology.orchestra.run.smithi174.stdout: debug ms = 1 2023-09-22T13:35:44.853 INFO:teuthology.orchestra.run.smithi174.stdout: debug mgr = 20 2023-09-22T13:35:44.853 INFO:teuthology.orchestra.run.smithi174.stdout: debug mon = 20 2023-09-22T13:35:44.853 INFO:teuthology.orchestra.run.smithi174.stdout: debug auth = 20 2023-09-22T13:35:44.853 INFO:teuthology.orchestra.run.smithi174.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout: mgr/telemetry/nag = false 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout:[mon] 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout: debug ms = 1 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout: debug mon = 20 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout: debug paxos = 20 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout: debug auth = 20 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout: mon data avail warn = 5 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout: mon mgr mkfs grace = 240 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:35:44.854 INFO:teuthology.orchestra.run.smithi174.stdout: mon osd reporter subtree level = osd 2023-09-22T13:35:44.855 INFO:teuthology.orchestra.run.smithi174.stdout: mon osd prime pg temp = true 2023-09-22T13:35:44.855 INFO:teuthology.orchestra.run.smithi174.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:35:44.855 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.855 INFO:teuthology.orchestra.run.smithi174.stdout: # rotate auth tickets quickly to exercise renewal paths 2023-09-22T13:35:44.855 INFO:teuthology.orchestra.run.smithi174.stdout: auth mon ticket ttl = 660 # 11m 2023-09-22T13:35:44.855 INFO:teuthology.orchestra.run.smithi174.stdout: auth service ticket ttl = 240 # 4m 2023-09-22T13:35:44.855 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.855 INFO:teuthology.orchestra.run.smithi174.stdout: # don't complain about insecure global_id in the test suite 2023-09-22T13:35:44.855 INFO:teuthology.orchestra.run.smithi174.stdout: mon_warn_on_insecure_global_id_reclaim = false 2023-09-22T13:35:44.855 INFO:teuthology.orchestra.run.smithi174.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2023-09-22T13:35:44.856 INFO:teuthology.orchestra.run.smithi174.stdout: mon osd crush smoke test = False 2023-09-22T13:35:44.856 INFO:teuthology.orchestra.run.smithi174.stdout: 2023-09-22T13:35:44.856 INFO:teuthology.orchestra.run.smithi174.stdout:[client] 2023-09-22T13:35:44.856 INFO:teuthology.orchestra.run.smithi174.stdout: rgw cache enabled = true 2023-09-22T13:35:44.856 INFO:teuthology.orchestra.run.smithi174.stdout: rgw enable ops log = true 2023-09-22T13:35:44.856 INFO:teuthology.orchestra.run.smithi174.stdout: rgw enable usage log = true 2023-09-22T13:35:44.856 INFO:teuthology.orchestra.run.smithi174.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2023-09-22T13:35:44.856 INFO:teuthology.orchestra.run.smithi174.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2023-09-22T13:35:44.856 INFO:teuthology.orchestra.run.smithi174.stdout: debug rgw = 20 2023-09-22T13:35:44.856 INFO:teuthology.orchestra.run.smithi174.stdout: rgw crypt require ssl = False 2023-09-22T13:35:44.856 INFO:teuthology.orchestra.run.smithi174.stdout: rgw crypt s3 kms backend = testing 2023-09-22T13:35:44.857 INFO:teuthology.orchestra.run.smithi174.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= 2023-09-22T13:35:44.857 INFO:teuthology.orchestra.run.smithi174.stdout: rgw lc debug interval = 10 2023-09-22T13:35:44.857 INFO:teuthology.orchestra.run.smithi174.stdout: rgw max chunk size = 4194304 2023-09-22T13:35:44.857 INFO:teuthology.orchestra.run.smithi174.stdout: rgw obj stripe size = 6291456 2023-09-22T13:35:44.857 INFO:teuthology.orchestra.run.smithi174.stdout: setgroup = ceph 2023-09-22T13:35:44.857 INFO:teuthology.orchestra.run.smithi174.stdout: setuser = ceph 2023-09-22T13:35:44.857 INFO:teuthology.orchestra.run.smithi174.stdout:[mon.a] 2023-09-22T13:35:44.857 INFO:teuthology.orchestra.run.smithi174.stdout:[mon.c] 2023-09-22T13:35:44.857 INFO:teuthology.orchestra.run.smithi174.stdout:[mon.b] 2023-09-22T13:35:44.858 INFO:tasks.ceph:Creating admin key on mon.a... 2023-09-22T13:35:44.859 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/ceph.keyring 2023-09-22T13:35:44.950 INFO:tasks.ceph:Copying monmap to all nodes... 2023-09-22T13:35:44.951 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:35:44.951 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/etc/ceph/ceph.keyring of=/dev/stdout 2023-09-22T13:35:44.968 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:35:44.968 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/home/ubuntu/cephtest/ceph.monmap of=/dev/stdout 2023-09-22T13:35:45.026 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi069.front.sepia.ceph.com 2023-09-22T13:35:45.026 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:35:45.026 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/ceph/ceph.keyring 2023-09-22T13:35:45.026 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod 0644 /etc/ceph/ceph.keyring 2023-09-22T13:35:45.102 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:35:45.102 DEBUG:teuthology.orchestra.run.smithi069:> dd of=/home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:35:45.159 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi174.front.sepia.ceph.com 2023-09-22T13:35:45.159 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:35:45.159 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/ceph/ceph.keyring 2023-09-22T13:35:45.159 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod 0644 /etc/ceph/ceph.keyring 2023-09-22T13:35:45.197 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:35:45.197 DEBUG:teuthology.orchestra.run.smithi174:> dd of=/home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:35:45.256 INFO:tasks.ceph:Setting up mon nodes... 2023-09-22T13:35:45.256 INFO:tasks.ceph:Setting up mgr nodes... 2023-09-22T13:35:45.256 DEBUG:teuthology.orchestra.run.smithi069:> sudo mkdir -p /var/lib/ceph/mgr/ceph-y && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.y /var/lib/ceph/mgr/ceph-y/keyring 2023-09-22T13:35:45.305 INFO:teuthology.orchestra.run.smithi069.stdout:creating /var/lib/ceph/mgr/ceph-y/keyring 2023-09-22T13:35:45.307 DEBUG:teuthology.orchestra.run.smithi174:> sudo mkdir -p /var/lib/ceph/mgr/ceph-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/ceph-x/keyring 2023-09-22T13:35:45.434 INFO:teuthology.orchestra.run.smithi174.stdout:creating /var/lib/ceph/mgr/ceph-x/keyring 2023-09-22T13:35:45.436 INFO:tasks.ceph:Setting up mds nodes... 2023-09-22T13:35:45.436 INFO:tasks.ceph_client:Setting up client nodes... 2023-09-22T13:35:45.436 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/ceph.client.0.keyring && sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-22T13:35:45.476 INFO:teuthology.orchestra.run.smithi069.stdout:creating /etc/ceph/ceph.client.0.keyring 2023-09-22T13:35:45.488 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/ceph.client.1.keyring && sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-22T13:35:45.527 INFO:teuthology.orchestra.run.smithi174.stdout:creating /etc/ceph/ceph.client.1.keyring 2023-09-22T13:35:45.539 INFO:tasks.ceph:Running mkfs on osd nodes... 2023-09-22T13:35:45.540 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi069.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi174.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2023-09-22T13:35:45.540 DEBUG:teuthology.orchestra.run.smithi069:> sudo mkdir -p /var/lib/ceph/osd/ceph-0 2023-09-22T13:35:45.567 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:35:45.567 INFO:tasks.ceph:role: osd.0 2023-09-22T13:35:45.567 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi069.front.sepia.ceph.com 2023-09-22T13:35:45.567 DEBUG:teuthology.orchestra.run.smithi069:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2023-09-22T13:35:45.642 INFO:teuthology.orchestra.run.smithi069.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:35:45.642 INFO:teuthology.orchestra.run.smithi069.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:35:45.642 INFO:teuthology.orchestra.run.smithi069.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:35:45.642 INFO:teuthology.orchestra.run.smithi069.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:35:45.642 INFO:teuthology.orchestra.run.smithi069.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:35:45.643 INFO:teuthology.orchestra.run.smithi069.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:35:45.643 INFO:teuthology.orchestra.run.smithi069.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:35:45.643 INFO:teuthology.orchestra.run.smithi069.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:35:45.643 INFO:teuthology.orchestra.run.smithi069.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:35:45.643 INFO:teuthology.orchestra.run.smithi069.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:35:46.077 INFO:teuthology.orchestra.run.smithi069.stdout:Discarding blocks...Done. 2023-09-22T13:35:46.077 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi069.front.sepia.ceph.com -o noatime 2023-09-22T13:35:46.078 DEBUG:teuthology.orchestra.run.smithi069:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-0 2023-09-22T13:35:46.230 DEBUG:teuthology.orchestra.run.smithi069:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-0 2023-09-22T13:35:46.267 DEBUG:teuthology.orchestra.run.smithi069:> sudo mkdir -p /var/lib/ceph/osd/ceph-1 2023-09-22T13:35:46.336 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:35:46.336 INFO:tasks.ceph:role: osd.1 2023-09-22T13:35:46.336 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi069.front.sepia.ceph.com 2023-09-22T13:35:46.336 DEBUG:teuthology.orchestra.run.smithi069:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2023-09-22T13:35:46.411 INFO:teuthology.orchestra.run.smithi069.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:35:46.412 INFO:teuthology.orchestra.run.smithi069.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:35:46.412 INFO:teuthology.orchestra.run.smithi069.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:35:46.412 INFO:teuthology.orchestra.run.smithi069.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:35:46.412 INFO:teuthology.orchestra.run.smithi069.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:35:46.412 INFO:teuthology.orchestra.run.smithi069.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:35:46.412 INFO:teuthology.orchestra.run.smithi069.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:35:46.412 INFO:teuthology.orchestra.run.smithi069.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:35:46.412 INFO:teuthology.orchestra.run.smithi069.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:35:46.412 INFO:teuthology.orchestra.run.smithi069.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:35:46.832 INFO:teuthology.orchestra.run.smithi069.stdout:Discarding blocks...Done. 2023-09-22T13:35:47.085 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi069.front.sepia.ceph.com -o noatime 2023-09-22T13:35:47.085 DEBUG:teuthology.orchestra.run.smithi069:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-1 2023-09-22T13:35:47.192 DEBUG:teuthology.orchestra.run.smithi069:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-1 2023-09-22T13:35:47.225 DEBUG:teuthology.orchestra.run.smithi069:> sudo mkdir -p /var/lib/ceph/osd/ceph-2 2023-09-22T13:35:47.294 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:35:47.294 INFO:tasks.ceph:role: osd.2 2023-09-22T13:35:47.294 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi069.front.sepia.ceph.com 2023-09-22T13:35:47.294 DEBUG:teuthology.orchestra.run.smithi069:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2023-09-22T13:35:47.369 INFO:teuthology.orchestra.run.smithi069.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:35:47.370 INFO:teuthology.orchestra.run.smithi069.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:35:47.370 INFO:teuthology.orchestra.run.smithi069.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:35:47.370 INFO:teuthology.orchestra.run.smithi069.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:35:47.370 INFO:teuthology.orchestra.run.smithi069.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:35:47.370 INFO:teuthology.orchestra.run.smithi069.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:35:47.370 INFO:teuthology.orchestra.run.smithi069.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:35:47.370 INFO:teuthology.orchestra.run.smithi069.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:35:47.370 INFO:teuthology.orchestra.run.smithi069.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:35:47.370 INFO:teuthology.orchestra.run.smithi069.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:35:47.814 INFO:teuthology.orchestra.run.smithi069.stdout:Discarding blocks...Done. 2023-09-22T13:35:47.815 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi069.front.sepia.ceph.com -o noatime 2023-09-22T13:35:47.816 DEBUG:teuthology.orchestra.run.smithi069:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-2 2023-09-22T13:35:47.963 DEBUG:teuthology.orchestra.run.smithi069:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-2 2023-09-22T13:35:47.994 DEBUG:teuthology.orchestra.run.smithi069:> sudo mkdir -p /var/lib/ceph/osd/ceph-3 2023-09-22T13:35:48.062 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:35:48.062 INFO:tasks.ceph:role: osd.3 2023-09-22T13:35:48.062 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi069.front.sepia.ceph.com 2023-09-22T13:35:48.062 DEBUG:teuthology.orchestra.run.smithi069:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2023-09-22T13:35:48.137 INFO:teuthology.orchestra.run.smithi069.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:35:48.137 INFO:teuthology.orchestra.run.smithi069.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:35:48.137 INFO:teuthology.orchestra.run.smithi069.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:35:48.137 INFO:teuthology.orchestra.run.smithi069.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:35:48.137 INFO:teuthology.orchestra.run.smithi069.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:35:48.137 INFO:teuthology.orchestra.run.smithi069.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:35:48.138 INFO:teuthology.orchestra.run.smithi069.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:35:48.138 INFO:teuthology.orchestra.run.smithi069.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:35:48.138 INFO:teuthology.orchestra.run.smithi069.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:35:48.138 INFO:teuthology.orchestra.run.smithi069.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:35:48.573 INFO:teuthology.orchestra.run.smithi069.stdout:Discarding blocks...Done. 2023-09-22T13:35:48.575 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi069.front.sepia.ceph.com -o noatime 2023-09-22T13:35:48.575 DEBUG:teuthology.orchestra.run.smithi069:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-3 2023-09-22T13:35:48.671 DEBUG:teuthology.orchestra.run.smithi069:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-3 2023-09-22T13:35:48.706 DEBUG:teuthology.orchestra.run.smithi069:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:35:48.796 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:48.794+0000 7fb30805b200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2023-09-22T13:35:48.796 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:48.795+0000 7fb30805b200 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2023-09-22T13:35:48.796 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:48.795+0000 7fb30805b200 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2023-09-22T13:35:48.796 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:48.795+0000 7fb30805b200 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2023-09-22T13:35:48.796 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:48.795+0000 7fb30805b200 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2023-09-22T13:35:48.796 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:48.795+0000 7fb30805b200 -1 bluestore(/var/lib/ceph/osd/ceph-0) _read_fsid unparsable uuid 2023-09-22T13:35:51.639 DEBUG:teuthology.orchestra.run.smithi069:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-22T13:35:51.673 DEBUG:teuthology.orchestra.run.smithi069:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:35:51.761 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:51.760+0000 7fabd2718200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2023-09-22T13:35:51.761 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:51.760+0000 7fabd2718200 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2023-09-22T13:35:51.761 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:51.760+0000 7fabd2718200 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2023-09-22T13:35:51.761 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:51.760+0000 7fabd2718200 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2023-09-22T13:35:51.762 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:51.760+0000 7fabd2718200 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2023-09-22T13:35:51.762 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:51.761+0000 7fabd2718200 -1 bluestore(/var/lib/ceph/osd/ceph-1) _read_fsid unparsable uuid 2023-09-22T13:35:54.604 DEBUG:teuthology.orchestra.run.smithi069:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-22T13:35:54.637 DEBUG:teuthology.orchestra.run.smithi069:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:35:54.726 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:54.725+0000 7f31cb147200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2023-09-22T13:35:54.726 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:54.725+0000 7f31cb147200 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2023-09-22T13:35:54.726 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:54.725+0000 7f31cb147200 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2023-09-22T13:35:54.726 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:54.725+0000 7f31cb147200 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2023-09-22T13:35:54.726 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:54.725+0000 7f31cb147200 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2023-09-22T13:35:54.726 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:54.726+0000 7f31cb147200 -1 bluestore(/var/lib/ceph/osd/ceph-2) _read_fsid unparsable uuid 2023-09-22T13:35:57.573 DEBUG:teuthology.orchestra.run.smithi069:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-22T13:35:57.606 DEBUG:teuthology.orchestra.run.smithi069:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:35:57.694 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:57.693+0000 7fdf9be78200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2023-09-22T13:35:57.694 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:57.693+0000 7fdf9be78200 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2023-09-22T13:35:57.694 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:57.693+0000 7fdf9be78200 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2023-09-22T13:35:57.694 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:57.693+0000 7fdf9be78200 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2023-09-22T13:35:57.695 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:57.693+0000 7fdf9be78200 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2023-09-22T13:35:57.695 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:35:57.694+0000 7fdf9be78200 -1 bluestore(/var/lib/ceph/osd/ceph-3) _read_fsid unparsable uuid 2023-09-22T13:36:00.534 DEBUG:teuthology.orchestra.run.smithi069:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-22T13:36:00.566 DEBUG:teuthology.orchestra.run.smithi174:> sudo mkdir -p /var/lib/ceph/osd/ceph-4 2023-09-22T13:36:00.599 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:36:00.599 INFO:tasks.ceph:role: osd.4 2023-09-22T13:36:00.599 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi174.front.sepia.ceph.com 2023-09-22T13:36:00.600 DEBUG:teuthology.orchestra.run.smithi174:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2023-09-22T13:36:00.678 INFO:teuthology.orchestra.run.smithi174.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:36:00.678 INFO:teuthology.orchestra.run.smithi174.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:36:00.678 INFO:teuthology.orchestra.run.smithi174.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:36:00.678 INFO:teuthology.orchestra.run.smithi174.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:36:00.679 INFO:teuthology.orchestra.run.smithi174.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:36:00.679 INFO:teuthology.orchestra.run.smithi174.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:36:00.679 INFO:teuthology.orchestra.run.smithi174.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:36:00.679 INFO:teuthology.orchestra.run.smithi174.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:36:00.679 INFO:teuthology.orchestra.run.smithi174.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:36:00.679 INFO:teuthology.orchestra.run.smithi174.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:36:01.107 INFO:teuthology.orchestra.run.smithi174.stdout:Discarding blocks...Done. 2023-09-22T13:36:01.108 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi174.front.sepia.ceph.com -o noatime 2023-09-22T13:36:01.109 DEBUG:teuthology.orchestra.run.smithi174:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-4 2023-09-22T13:36:01.274 DEBUG:teuthology.orchestra.run.smithi174:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-4 2023-09-22T13:36:01.314 DEBUG:teuthology.orchestra.run.smithi174:> sudo mkdir -p /var/lib/ceph/osd/ceph-5 2023-09-22T13:36:01.385 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:36:01.385 INFO:tasks.ceph:role: osd.5 2023-09-22T13:36:01.385 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi174.front.sepia.ceph.com 2023-09-22T13:36:01.385 DEBUG:teuthology.orchestra.run.smithi174:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2023-09-22T13:36:01.461 INFO:teuthology.orchestra.run.smithi174.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:36:01.462 INFO:teuthology.orchestra.run.smithi174.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:36:01.462 INFO:teuthology.orchestra.run.smithi174.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:36:01.462 INFO:teuthology.orchestra.run.smithi174.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:36:01.462 INFO:teuthology.orchestra.run.smithi174.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:36:01.462 INFO:teuthology.orchestra.run.smithi174.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:36:01.462 INFO:teuthology.orchestra.run.smithi174.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:36:01.462 INFO:teuthology.orchestra.run.smithi174.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:36:01.462 INFO:teuthology.orchestra.run.smithi174.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:36:01.462 INFO:teuthology.orchestra.run.smithi174.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:36:01.894 INFO:teuthology.orchestra.run.smithi174.stdout:Discarding blocks...Done. 2023-09-22T13:36:01.896 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi174.front.sepia.ceph.com -o noatime 2023-09-22T13:36:01.896 DEBUG:teuthology.orchestra.run.smithi174:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-5 2023-09-22T13:36:02.006 DEBUG:teuthology.orchestra.run.smithi174:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-5 2023-09-22T13:36:02.040 DEBUG:teuthology.orchestra.run.smithi174:> sudo mkdir -p /var/lib/ceph/osd/ceph-6 2023-09-22T13:36:02.109 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:36:02.109 INFO:tasks.ceph:role: osd.6 2023-09-22T13:36:02.109 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi174.front.sepia.ceph.com 2023-09-22T13:36:02.109 DEBUG:teuthology.orchestra.run.smithi174:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2023-09-22T13:36:02.185 INFO:teuthology.orchestra.run.smithi174.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:36:02.185 INFO:teuthology.orchestra.run.smithi174.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:36:02.185 INFO:teuthology.orchestra.run.smithi174.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:36:02.185 INFO:teuthology.orchestra.run.smithi174.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:36:02.185 INFO:teuthology.orchestra.run.smithi174.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:36:02.185 INFO:teuthology.orchestra.run.smithi174.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:36:02.185 INFO:teuthology.orchestra.run.smithi174.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:36:02.185 INFO:teuthology.orchestra.run.smithi174.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:36:02.186 INFO:teuthology.orchestra.run.smithi174.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:36:02.186 INFO:teuthology.orchestra.run.smithi174.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:36:02.629 INFO:teuthology.orchestra.run.smithi174.stdout:Discarding blocks...Done. 2023-09-22T13:36:02.631 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi174.front.sepia.ceph.com -o noatime 2023-09-22T13:36:02.631 DEBUG:teuthology.orchestra.run.smithi174:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-6 2023-09-22T13:36:02.788 DEBUG:teuthology.orchestra.run.smithi174:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-6 2023-09-22T13:36:02.823 DEBUG:teuthology.orchestra.run.smithi174:> sudo mkdir -p /var/lib/ceph/osd/ceph-7 2023-09-22T13:36:02.894 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:36:02.894 INFO:tasks.ceph:role: osd.7 2023-09-22T13:36:02.894 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi174.front.sepia.ceph.com 2023-09-22T13:36:02.894 DEBUG:teuthology.orchestra.run.smithi174:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2023-09-22T13:36:02.970 INFO:teuthology.orchestra.run.smithi174.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:36:02.970 INFO:teuthology.orchestra.run.smithi174.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:36:02.970 INFO:teuthology.orchestra.run.smithi174.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:36:02.970 INFO:teuthology.orchestra.run.smithi174.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:36:02.970 INFO:teuthology.orchestra.run.smithi174.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:36:02.970 INFO:teuthology.orchestra.run.smithi174.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:36:02.970 INFO:teuthology.orchestra.run.smithi174.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:36:02.971 INFO:teuthology.orchestra.run.smithi174.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:36:02.971 INFO:teuthology.orchestra.run.smithi174.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:36:02.971 INFO:teuthology.orchestra.run.smithi174.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:36:03.415 INFO:teuthology.orchestra.run.smithi174.stdout:Discarding blocks...Done. 2023-09-22T13:36:03.417 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi174.front.sepia.ceph.com -o noatime 2023-09-22T13:36:03.417 DEBUG:teuthology.orchestra.run.smithi174:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-7 2023-09-22T13:36:03.563 DEBUG:teuthology.orchestra.run.smithi174:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-7 2023-09-22T13:36:03.601 DEBUG:teuthology.orchestra.run.smithi174:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 4 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:36:03.701 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:03.699+0000 7f750d656200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-4/keyring: can't open /var/lib/ceph/osd/ceph-4/keyring: (2) No such file or directory 2023-09-22T13:36:03.701 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:03.700+0000 7f750d656200 -1 created new key in keyring /var/lib/ceph/osd/ceph-4/keyring 2023-09-22T13:36:03.701 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:03.700+0000 7f750d656200 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2023-09-22T13:36:03.701 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:03.700+0000 7f750d656200 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2023-09-22T13:36:03.701 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:03.700+0000 7f750d656200 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2023-09-22T13:36:03.702 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:03.700+0000 7f750d656200 -1 bluestore(/var/lib/ceph/osd/ceph-4) _read_fsid unparsable uuid 2023-09-22T13:36:06.539 DEBUG:teuthology.orchestra.run.smithi174:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-09-22T13:36:06.569 DEBUG:teuthology.orchestra.run.smithi174:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 5 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:36:06.657 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:06.656+0000 7f58a20ff200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-5/keyring: can't open /var/lib/ceph/osd/ceph-5/keyring: (2) No such file or directory 2023-09-22T13:36:06.657 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:06.656+0000 7f58a20ff200 -1 created new key in keyring /var/lib/ceph/osd/ceph-5/keyring 2023-09-22T13:36:06.657 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:06.656+0000 7f58a20ff200 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2023-09-22T13:36:06.657 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:06.656+0000 7f58a20ff200 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2023-09-22T13:36:06.658 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:06.656+0000 7f58a20ff200 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2023-09-22T13:36:06.658 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:06.657+0000 7f58a20ff200 -1 bluestore(/var/lib/ceph/osd/ceph-5) _read_fsid unparsable uuid 2023-09-22T13:36:09.500 DEBUG:teuthology.orchestra.run.smithi174:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-09-22T13:36:09.533 DEBUG:teuthology.orchestra.run.smithi174:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 6 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:36:09.620 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:09.619+0000 7f72f224c200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-6/keyring: can't open /var/lib/ceph/osd/ceph-6/keyring: (2) No such file or directory 2023-09-22T13:36:09.620 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:09.619+0000 7f72f224c200 -1 created new key in keyring /var/lib/ceph/osd/ceph-6/keyring 2023-09-22T13:36:09.620 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:09.619+0000 7f72f224c200 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2023-09-22T13:36:09.620 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:09.619+0000 7f72f224c200 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2023-09-22T13:36:09.620 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:09.619+0000 7f72f224c200 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2023-09-22T13:36:09.620 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:09.619+0000 7f72f224c200 -1 bluestore(/var/lib/ceph/osd/ceph-6) _read_fsid unparsable uuid 2023-09-22T13:36:12.471 DEBUG:teuthology.orchestra.run.smithi174:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-22T13:36:12.502 DEBUG:teuthology.orchestra.run.smithi174:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 7 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:36:12.591 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:12.590+0000 7f2351bc8200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-7/keyring: can't open /var/lib/ceph/osd/ceph-7/keyring: (2) No such file or directory 2023-09-22T13:36:12.591 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:12.590+0000 7f2351bc8200 -1 created new key in keyring /var/lib/ceph/osd/ceph-7/keyring 2023-09-22T13:36:12.591 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:12.590+0000 7f2351bc8200 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2023-09-22T13:36:12.591 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:12.590+0000 7f2351bc8200 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2023-09-22T13:36:12.592 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:12.590+0000 7f2351bc8200 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2023-09-22T13:36:12.592 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:36:12.591+0000 7f2351bc8200 -1 bluestore(/var/lib/ceph/osd/ceph-7) _read_fsid unparsable uuid 2023-09-22T13:36:15.430 DEBUG:teuthology.orchestra.run.smithi174:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-09-22T13:36:15.462 INFO:tasks.ceph:Reading keys from all nodes... 2023-09-22T13:36:15.462 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:15.463 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/mgr/ceph-y/keyring of=/dev/stdout 2023-09-22T13:36:15.495 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:15.495 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/osd/ceph-0/keyring of=/dev/stdout 2023-09-22T13:36:15.567 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:15.567 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/osd/ceph-1/keyring of=/dev/stdout 2023-09-22T13:36:15.638 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:15.638 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/osd/ceph-2/keyring of=/dev/stdout 2023-09-22T13:36:15.707 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:15.707 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/osd/ceph-3/keyring of=/dev/stdout 2023-09-22T13:36:15.776 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:36:15.776 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/var/lib/ceph/mgr/ceph-x/keyring of=/dev/stdout 2023-09-22T13:36:15.808 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:36:15.808 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/var/lib/ceph/osd/ceph-4/keyring of=/dev/stdout 2023-09-22T13:36:15.880 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:36:15.880 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/var/lib/ceph/osd/ceph-5/keyring of=/dev/stdout 2023-09-22T13:36:15.951 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:36:15.951 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/var/lib/ceph/osd/ceph-6/keyring of=/dev/stdout 2023-09-22T13:36:16.019 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:36:16.020 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/var/lib/ceph/osd/ceph-7/keyring of=/dev/stdout 2023-09-22T13:36:16.088 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:16.088 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/etc/ceph/ceph.client.0.keyring of=/dev/stdout 2023-09-22T13:36:16.107 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:36:16.107 DEBUG:teuthology.orchestra.run.smithi174:> dd if=/etc/ceph/ceph.client.1.keyring of=/dev/stdout 2023-09-22T13:36:16.146 INFO:tasks.ceph:Adding keys to all mons... 2023-09-22T13:36:16.147 DEBUG:teuthology.orchestra.run.smithi069:> sudo tee -a /etc/ceph/ceph.keyring 2023-09-22T13:36:16.149 DEBUG:teuthology.orchestra.run.smithi174:> sudo tee -a /etc/ceph/ceph.keyring 2023-09-22T13:36:16.189 INFO:teuthology.orchestra.run.smithi069.stdout:[mgr.y] 2023-09-22T13:36:16.190 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQAxmA1l+RI7EhAAL5byDb7+/+E8d0VEUBtOBA== 2023-09-22T13:36:16.190 INFO:teuthology.orchestra.run.smithi069.stdout:[osd.0] 2023-09-22T13:36:16.190 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQA0mA1lY7hsLxAAq+4e+LKT23L8idYgEJup+w== 2023-09-22T13:36:16.190 INFO:teuthology.orchestra.run.smithi069.stdout:[osd.1] 2023-09-22T13:36:16.190 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQA3mA1lwqNdLRAA1UgTaQdblQIlNP2jVaZUOA== 2023-09-22T13:36:16.190 INFO:teuthology.orchestra.run.smithi069.stdout:[osd.2] 2023-09-22T13:36:16.190 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQA6mA1l1PhFKxAAVxE1PfMt6zird487AOmgNg== 2023-09-22T13:36:16.190 INFO:teuthology.orchestra.run.smithi069.stdout:[osd.3] 2023-09-22T13:36:16.190 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQA9mA1lPXhiKRAACQTgNqccKIG3ye78X9OxgQ== 2023-09-22T13:36:16.191 INFO:teuthology.orchestra.run.smithi069.stdout:[mgr.x] 2023-09-22T13:36:16.191 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQAxmA1lzXDjGRAAJpXyYkGph2yKdNFj55Bqsw== 2023-09-22T13:36:16.191 INFO:teuthology.orchestra.run.smithi069.stdout:[osd.4] 2023-09-22T13:36:16.191 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQBDmA1lNIrIKRAAr6qizfHGnT6iGe1Pz9quig== 2023-09-22T13:36:16.191 INFO:teuthology.orchestra.run.smithi069.stdout:[osd.5] 2023-09-22T13:36:16.191 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQBGmA1lgxUuJxAA33xjBw7UZ798nQnVci3B+w== 2023-09-22T13:36:16.191 INFO:teuthology.orchestra.run.smithi069.stdout:[osd.6] 2023-09-22T13:36:16.191 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQBJmA1lAQD1JBAAERFCV+iSo1oelAYk92nHCw== 2023-09-22T13:36:16.191 INFO:teuthology.orchestra.run.smithi069.stdout:[osd.7] 2023-09-22T13:36:16.191 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQBMmA1laFpAIxAANiyzEA5fT8OMq3s4C2+1Dw== 2023-09-22T13:36:16.192 INFO:teuthology.orchestra.run.smithi069.stdout:[client.0] 2023-09-22T13:36:16.192 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQAxmA1lEfljHBAAgUi9jYpJlBPSPWS/yGAw1g== 2023-09-22T13:36:16.192 INFO:teuthology.orchestra.run.smithi069.stdout:[client.1] 2023-09-22T13:36:16.192 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQAxmA1lLpZ5HxAAdpaoNZfxyDn/cx4Kqniglw== 2023-09-22T13:36:16.212 INFO:teuthology.orchestra.run.smithi174.stdout:[mgr.y] 2023-09-22T13:36:16.213 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQAxmA1l+RI7EhAAL5byDb7+/+E8d0VEUBtOBA== 2023-09-22T13:36:16.213 INFO:teuthology.orchestra.run.smithi174.stdout:[osd.0] 2023-09-22T13:36:16.213 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQA0mA1lY7hsLxAAq+4e+LKT23L8idYgEJup+w== 2023-09-22T13:36:16.213 INFO:teuthology.orchestra.run.smithi174.stdout:[osd.1] 2023-09-22T13:36:16.213 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQA3mA1lwqNdLRAA1UgTaQdblQIlNP2jVaZUOA== 2023-09-22T13:36:16.213 INFO:teuthology.orchestra.run.smithi174.stdout:[osd.2] 2023-09-22T13:36:16.213 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQA6mA1l1PhFKxAAVxE1PfMt6zird487AOmgNg== 2023-09-22T13:36:16.213 INFO:teuthology.orchestra.run.smithi174.stdout:[osd.3] 2023-09-22T13:36:16.213 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQA9mA1lPXhiKRAACQTgNqccKIG3ye78X9OxgQ== 2023-09-22T13:36:16.213 INFO:teuthology.orchestra.run.smithi174.stdout:[mgr.x] 2023-09-22T13:36:16.214 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQAxmA1lzXDjGRAAJpXyYkGph2yKdNFj55Bqsw== 2023-09-22T13:36:16.214 INFO:teuthology.orchestra.run.smithi174.stdout:[osd.4] 2023-09-22T13:36:16.214 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQBDmA1lNIrIKRAAr6qizfHGnT6iGe1Pz9quig== 2023-09-22T13:36:16.214 INFO:teuthology.orchestra.run.smithi174.stdout:[osd.5] 2023-09-22T13:36:16.214 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQBGmA1lgxUuJxAA33xjBw7UZ798nQnVci3B+w== 2023-09-22T13:36:16.214 INFO:teuthology.orchestra.run.smithi174.stdout:[osd.6] 2023-09-22T13:36:16.214 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQBJmA1lAQD1JBAAERFCV+iSo1oelAYk92nHCw== 2023-09-22T13:36:16.214 INFO:teuthology.orchestra.run.smithi174.stdout:[osd.7] 2023-09-22T13:36:16.214 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQBMmA1laFpAIxAANiyzEA5fT8OMq3s4C2+1Dw== 2023-09-22T13:36:16.214 INFO:teuthology.orchestra.run.smithi174.stdout:[client.0] 2023-09-22T13:36:16.215 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQAxmA1lEfljHBAAgUi9jYpJlBPSPWS/yGAw1g== 2023-09-22T13:36:16.215 INFO:teuthology.orchestra.run.smithi174.stdout:[client.1] 2023-09-22T13:36:16.215 INFO:teuthology.orchestra.run.smithi174.stdout: key = AQAxmA1lLpZ5HxAAdpaoNZfxyDn/cx4Kqniglw== 2023-09-22T13:36:16.216 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:36:16.235 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:36:16.304 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.307 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.356 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.358 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.406 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.408 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.456 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.458 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.506 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:36:16.509 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:36:16.557 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.560 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.608 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.611 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.659 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.661 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.710 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.712 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:36:16.762 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:36:16.764 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:36:16.815 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:36:16.817 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:36:16.868 INFO:tasks.ceph:Running mkfs on mon nodes... 2023-09-22T13:36:16.868 DEBUG:teuthology.orchestra.run.smithi069:> sudo mkdir -p /var/lib/ceph/mon/ceph-a 2023-09-22T13:36:16.894 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i a --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2023-09-22T13:36:16.988 DEBUG:teuthology.orchestra.run.smithi069:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-a 2023-09-22T13:36:17.014 DEBUG:teuthology.orchestra.run.smithi069:> sudo mkdir -p /var/lib/ceph/mon/ceph-c 2023-09-22T13:36:17.082 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i c --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2023-09-22T13:36:17.175 DEBUG:teuthology.orchestra.run.smithi069:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-c 2023-09-22T13:36:17.202 DEBUG:teuthology.orchestra.run.smithi174:> sudo mkdir -p /var/lib/ceph/mon/ceph-b 2023-09-22T13:36:17.229 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i b --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2023-09-22T13:36:17.323 DEBUG:teuthology.orchestra.run.smithi174:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-b 2023-09-22T13:36:17.350 DEBUG:teuthology.orchestra.run.smithi069:> rm -- /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:36:17.352 DEBUG:teuthology.orchestra.run.smithi174:> rm -- /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:36:17.408 INFO:tasks.ceph:Starting mon daemons in cluster ceph... 2023-09-22T13:36:17.409 INFO:tasks.ceph.mon.a:Restarting daemon 2023-09-22T13:36:17.409 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2023-09-22T13:36:17.412 INFO:tasks.ceph.mon.a:Started 2023-09-22T13:36:17.412 INFO:tasks.ceph.mon.c:Restarting daemon 2023-09-22T13:36:17.412 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2023-09-22T13:36:17.414 INFO:tasks.ceph.mon.c:Started 2023-09-22T13:36:17.414 INFO:tasks.ceph.mon.b:Restarting daemon 2023-09-22T13:36:17.415 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2023-09-22T13:36:17.451 INFO:tasks.ceph.mon.b:Started 2023-09-22T13:36:17.451 INFO:tasks.ceph:Starting mgr daemons in cluster ceph... 2023-09-22T13:36:17.451 INFO:tasks.ceph.mgr.y:Restarting daemon 2023-09-22T13:36:17.451 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2023-09-22T13:36:17.453 INFO:tasks.ceph.mgr.y:Started 2023-09-22T13:36:17.454 INFO:tasks.ceph.mgr.x:Restarting daemon 2023-09-22T13:36:17.454 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2023-09-22T13:36:17.456 INFO:tasks.ceph.mgr.x:Started 2023-09-22T13:36:17.456 INFO:tasks.ceph:Setting crush tunables to default 2023-09-22T13:36:17.456 DEBUG:teuthology.orchestra.run.smithi069:> sudo ceph --cluster ceph osd crush tunables default 2023-09-22T13:36:22.862 INFO:teuthology.orchestra.run.smithi069.stderr:adjusted tunables profile to default 2023-09-22T13:36:22.873 INFO:tasks.ceph:Starting osd daemons in cluster ceph... 2023-09-22T13:36:22.874 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:22.874 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/osd/ceph-0/fsid of=/dev/stdout 2023-09-22T13:36:22.900 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:22.900 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/osd/ceph-1/fsid of=/dev/stdout 2023-09-22T13:36:22.902 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:22.901+0000 7f030fe03680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-22T13:36:22.911 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:22.910+0000 7fb1336bf680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-22T13:36:22.939 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:22.939 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/osd/ceph-2/fsid of=/dev/stdout 2023-09-22T13:36:22.970 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:22.970+0000 7f030fe03680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-22T13:36:22.985 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:22.985+0000 7fb1336bf680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-22T13:36:23.007 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:23.007 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/osd/ceph-3/fsid of=/dev/stdout 2023-09-22T13:36:23.075 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:36:23.075 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/var/lib/ceph/osd/ceph-4/fsid of=/dev/stdout 2023-09-22T13:36:23.102 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:36:23.103 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/var/lib/ceph/osd/ceph-5/fsid of=/dev/stdout 2023-09-22T13:36:23.171 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:36:23.173 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/var/lib/ceph/osd/ceph-6/fsid of=/dev/stdout 2023-09-22T13:36:23.238 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2023-09-22T13:36:23.238 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/var/lib/ceph/osd/ceph-7/fsid of=/dev/stdout 2023-09-22T13:36:23.305 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph --cluster ceph osd new 0fcbd366-81e0-48ab-855b-9eee6e25c05f 0 2023-09-22T13:36:23.412 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:23.411+0000 7f030fe03680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-22T13:36:23.444 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:23.444+0000 7fb1336bf680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-22T13:36:23.553 INFO:teuthology.orchestra.run.smithi174.stdout:0 2023-09-22T13:36:23.565 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph --cluster ceph osd new 8277992f-0f9d-4f92-a61a-c9d4e8f1731b 1 2023-09-22T13:36:23.777 INFO:teuthology.orchestra.run.smithi174.stdout:1 2023-09-22T13:36:23.790 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph --cluster ceph osd new 78664a54-7349-4f5a-a1df-eaa61aee8b6a 2 2023-09-22T13:36:24.001 INFO:teuthology.orchestra.run.smithi174.stdout:2 2023-09-22T13:36:24.013 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph --cluster ceph osd new b38b9327-c129-462d-b5e4-79ba3e2b0118 3 2023-09-22T13:36:24.162 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:24.162+0000 7fb1336bf680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-22T13:36:24.171 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:24.170+0000 7f030fe03680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-22T13:36:24.230 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:24.229+0000 7fb1336bf680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-22T13:36:24.239 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:24.238+0000 7f030fe03680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-22T13:36:24.252 INFO:teuthology.orchestra.run.smithi174.stdout:3 2023-09-22T13:36:24.265 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph --cluster ceph osd new 7fffaf9b-1a75-43bd-8a39-1bc2efba2208 4 2023-09-22T13:36:24.416 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:24.416+0000 7fb1336bf680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-22T13:36:24.439 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:24.438+0000 7f030fe03680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-22T13:36:24.479 INFO:teuthology.orchestra.run.smithi174.stdout:4 2023-09-22T13:36:24.492 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph --cluster ceph osd new 3d45afee-adf7-43fb-912f-ca92d46d2758 5 2023-09-22T13:36:24.702 INFO:teuthology.orchestra.run.smithi174.stdout:5 2023-09-22T13:36:24.733 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph --cluster ceph osd new 3c517ba1-d327-487c-8c36-7f78528cba1d 6 2023-09-22T13:36:24.851 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:24.850+0000 7fb1336bf680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-22T13:36:24.902 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:24.901+0000 7f030fe03680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-22T13:36:24.945 INFO:teuthology.orchestra.run.smithi174.stdout:6 2023-09-22T13:36:24.954 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:24.954+0000 7fb1336bf680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:36:24.957 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph --cluster ceph osd new 3056e758-0438-4e6e-990b-f972f8a648f9 7 2023-09-22T13:36:25.010 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:25.009+0000 7f030fe03680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:36:25.105 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:25.105+0000 7fb1336bf680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-22T13:36:25.171 INFO:teuthology.orchestra.run.smithi174.stdout:7 2023-09-22T13:36:25.173 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:25.172+0000 7f030fe03680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-22T13:36:25.182 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:25.181+0000 7fb1336bf680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-22T13:36:25.184 INFO:tasks.ceph.osd.0:Restarting daemon 2023-09-22T13:36:25.184 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2023-09-22T13:36:25.186 INFO:tasks.ceph.osd.0:Started 2023-09-22T13:36:25.186 INFO:tasks.ceph.osd.1:Restarting daemon 2023-09-22T13:36:25.186 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2023-09-22T13:36:25.188 INFO:tasks.ceph.osd.1:Started 2023-09-22T13:36:25.188 INFO:tasks.ceph.osd.2:Restarting daemon 2023-09-22T13:36:25.188 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2023-09-22T13:36:25.190 INFO:tasks.ceph.osd.2:Started 2023-09-22T13:36:25.190 INFO:tasks.ceph.osd.3:Restarting daemon 2023-09-22T13:36:25.190 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2023-09-22T13:36:25.192 INFO:tasks.ceph.osd.3:Started 2023-09-22T13:36:25.192 INFO:tasks.ceph.osd.4:Restarting daemon 2023-09-22T13:36:25.193 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 4 2023-09-22T13:36:25.194 INFO:tasks.ceph.osd.4:Started 2023-09-22T13:36:25.195 INFO:tasks.ceph.osd.5:Restarting daemon 2023-09-22T13:36:25.195 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 5 2023-09-22T13:36:25.196 INFO:tasks.ceph.osd.5:Started 2023-09-22T13:36:25.197 INFO:tasks.ceph.osd.6:Restarting daemon 2023-09-22T13:36:25.197 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 6 2023-09-22T13:36:25.198 INFO:tasks.ceph.osd.6:Started 2023-09-22T13:36:25.198 INFO:tasks.ceph.osd.7:Restarting daemon 2023-09-22T13:36:25.199 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 7 2023-09-22T13:36:25.200 INFO:tasks.ceph.osd.7:Started 2023-09-22T13:36:25.201 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-09-22T13:36:25.264 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:25.264+0000 7f030fe03680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-22T13:36:25.286 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:25.285+0000 7fb1336bf680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-22T13:36:25.341 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:25.340+0000 7f030fe03680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-22T13:36:25.433 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:36:25.433 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":10,"fsid":"328c464b-5dfe-4280-b09b-dba594b52bd4","created":"2023-09-22T13:36:22.739115+0000","modified":"2023-09-22T13:36:25.164335+0000","last_up_change":"0.000000","last_in_change":"2023-09-22T13:36:25.164335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[],"osds":[{"osd":0,"uuid":"0fcbd366-81e0-48ab-855b-9eee6e25c05f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"8277992f-0f9d-4f92-a61a-c9d4e8f1731b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"78664a54-7349-4f5a-a1df-eaa61aee8b6a","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"b38b9327-c129-462d-b5e4-79ba3e2b0118","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"7fffaf9b-1a75-43bd-8a39-1bc2efba2208","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"3d45afee-adf7-43fb-912f-ca92d46d2758","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"3c517ba1-d327-487c-8c36-7f78528cba1d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"3056e758-0438-4e6e-990b-f972f8a648f9","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:36:25.445 INFO:tasks.ceph.ceph_manager.ceph:[] 2023-09-22T13:36:25.446 INFO:tasks.ceph:Waiting for OSDs to come up 2023-09-22T13:36:25.562 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:25.561+0000 7fb1336bf680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-22T13:36:25.602 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:25.601+0000 7f030fe03680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-22T13:36:25.757 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:25.756+0000 7fb1336bf680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-22T13:36:25.797 INFO:tasks.ceph.osd.4.smithi174.stderr:2023-09-22T13:36:25.797+0000 7fe02678c200 -1 Falling back to public interface 2023-09-22T13:36:25.799 INFO:tasks.ceph.osd.0.smithi069.stderr:2023-09-22T13:36:25.798+0000 7f70e77ef200 -1 Falling back to public interface 2023-09-22T13:36:25.800 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:25.799+0000 7f030fe03680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-22T13:36:25.803 INFO:tasks.ceph.osd.3.smithi069.stderr:2023-09-22T13:36:25.802+0000 7fb0a5010200 -1 Falling back to public interface 2023-09-22T13:36:25.806 INFO:tasks.ceph.osd.5.smithi174.stderr:2023-09-22T13:36:25.805+0000 7f561612b200 -1 Falling back to public interface 2023-09-22T13:36:25.818 INFO:tasks.ceph.osd.1.smithi069.stderr:2023-09-22T13:36:25.817+0000 7fa5a1e56200 -1 Falling back to public interface 2023-09-22T13:36:25.821 INFO:tasks.ceph.osd.2.smithi069.stderr:2023-09-22T13:36:25.821+0000 7f320f1f5200 -1 Falling back to public interface 2023-09-22T13:36:25.823 INFO:tasks.ceph.osd.7.smithi174.stderr:2023-09-22T13:36:25.822+0000 7fbd01288200 -1 Falling back to public interface 2023-09-22T13:36:25.828 INFO:tasks.ceph.osd.6.smithi174.stderr:2023-09-22T13:36:25.827+0000 7fb3ac4c1200 -1 Falling back to public interface 2023-09-22T13:36:26.251 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-22T13:36:26.429 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:26.429+0000 7fb1336bf680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-22T13:36:26.439 INFO:teuthology.misc.health.smithi069.stdout: 2023-09-22T13:36:26.439 INFO:teuthology.misc.health.smithi069.stdout:{"epoch":10,"fsid":"328c464b-5dfe-4280-b09b-dba594b52bd4","created":"2023-09-22T13:36:22.739115+0000","modified":"2023-09-22T13:36:25.164335+0000","last_up_change":"0.000000","last_in_change":"2023-09-22T13:36:25.164335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[],"osds":[{"osd":0,"uuid":"0fcbd366-81e0-48ab-855b-9eee6e25c05f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"8277992f-0f9d-4f92-a61a-c9d4e8f1731b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"78664a54-7349-4f5a-a1df-eaa61aee8b6a","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"b38b9327-c129-462d-b5e4-79ba3e2b0118","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"7fffaf9b-1a75-43bd-8a39-1bc2efba2208","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"3d45afee-adf7-43fb-912f-ca92d46d2758","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"3c517ba1-d327-487c-8c36-7f78528cba1d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"3056e758-0438-4e6e-990b-f972f8a648f9","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:36:26.450 DEBUG:teuthology.misc:0 of 8 OSDs are up 2023-09-22T13:36:26.478 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:26.477+0000 7f030fe03680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-22T13:36:26.500 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:26.499+0000 7fb1336bf680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-22T13:36:26.547 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:26.545+0000 7f030fe03680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-22T13:36:26.577 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:26.577+0000 7fb1336bf680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-22T13:36:26.625 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:26.624+0000 7f030fe03680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-22T13:36:26.725 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:26.724+0000 7fb1336bf680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-22T13:36:26.775 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:26.774+0000 7f030fe03680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-22T13:36:26.795 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:26.795+0000 7fb1336bf680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-22T13:36:26.845 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:26.844+0000 7f030fe03680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-22T13:36:26.911 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:26.910+0000 7fb1336bf680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-22T13:36:26.963 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:26.962+0000 7f030fe03680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-22T13:36:27.017 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:27.017+0000 7fb1336bf680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:36:27.072 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:27.071+0000 7f030fe03680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:36:27.361 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:27.360+0000 7fb1336bf680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-22T13:36:27.397 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:27.397+0000 7f030fe03680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-22T13:36:27.432 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:27.431+0000 7fb1336bf680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-22T13:36:27.470 INFO:tasks.ceph.mgr.x.smithi174.stderr:2023-09-22T13:36:27.469+0000 7f030fe03680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-22T13:36:28.454 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:28.454+0000 7fb0e8bdb700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-22T13:36:28.656 INFO:tasks.ceph.mgr.y.smithi069.stderr:2023-09-22T13:36:28.656+0000 7fb0e8bdb700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-22T13:36:29.247 INFO:tasks.ceph.osd.4.smithi174.stderr:2023-09-22T13:36:29.246+0000 7fe02678c200 -1 osd.4 0 log_to_monitors {default=true} 2023-09-22T13:36:29.278 INFO:tasks.ceph.osd.3.smithi069.stderr:2023-09-22T13:36:29.277+0000 7fb0a5010200 -1 osd.3 0 log_to_monitors {default=true} 2023-09-22T13:36:29.302 INFO:tasks.ceph.osd.0.smithi069.stderr:2023-09-22T13:36:29.301+0000 7f70e77ef200 -1 osd.0 0 log_to_monitors {default=true} 2023-09-22T13:36:29.312 INFO:tasks.ceph.osd.2.smithi069.stderr:2023-09-22T13:36:29.312+0000 7f320f1f5200 -1 osd.2 0 log_to_monitors {default=true} 2023-09-22T13:36:29.326 INFO:tasks.ceph.osd.5.smithi174.stderr:2023-09-22T13:36:29.325+0000 7f561612b200 -1 osd.5 0 log_to_monitors {default=true} 2023-09-22T13:36:29.330 INFO:tasks.ceph.osd.1.smithi069.stderr:2023-09-22T13:36:29.330+0000 7fa5a1e56200 -1 osd.1 0 log_to_monitors {default=true} 2023-09-22T13:36:29.348 INFO:tasks.ceph.osd.7.smithi174.stderr:2023-09-22T13:36:29.347+0000 7fbd01288200 -1 osd.7 0 log_to_monitors {default=true} 2023-09-22T13:36:29.371 INFO:tasks.ceph.osd.6.smithi174.stderr:2023-09-22T13:36:29.370+0000 7fb3ac4c1200 -1 osd.6 0 log_to_monitors {default=true} 2023-09-22T13:36:30.459 INFO:tasks.ceph.osd.5.smithi174.stderr:2023-09-22T13:36:30.458+0000 7f560cacc700 -1 osd.5 0 waiting for initial osdmap 2023-09-22T13:36:30.459 INFO:tasks.ceph.osd.7.smithi174.stderr:2023-09-22T13:36:30.458+0000 7fbcf7c29700 -1 osd.7 0 waiting for initial osdmap 2023-09-22T13:36:30.461 INFO:tasks.ceph.osd.2.smithi069.stderr:2023-09-22T13:36:30.460+0000 7f3205b96700 -1 osd.2 0 waiting for initial osdmap 2023-09-22T13:36:30.463 INFO:tasks.ceph.osd.4.smithi174.stderr:2023-09-22T13:36:30.463+0000 7fe01d12d700 -1 osd.4 0 waiting for initial osdmap 2023-09-22T13:36:30.464 INFO:tasks.ceph.osd.6.smithi174.stderr:2023-09-22T13:36:30.463+0000 7fb3a2e62700 -1 osd.6 0 waiting for initial osdmap 2023-09-22T13:36:30.464 INFO:tasks.ceph.osd.0.smithi069.stderr:2023-09-22T13:36:30.464+0000 7f70de190700 -1 osd.0 0 waiting for initial osdmap 2023-09-22T13:36:30.465 INFO:tasks.ceph.osd.3.smithi069.stderr:2023-09-22T13:36:30.464+0000 7fb09b9b1700 -1 osd.3 0 waiting for initial osdmap 2023-09-22T13:36:30.465 INFO:tasks.ceph.osd.1.smithi069.stderr:2023-09-22T13:36:30.465+0000 7fa5987f7700 -1 osd.1 0 waiting for initial osdmap 2023-09-22T13:36:30.468 INFO:tasks.ceph.osd.2.smithi069.stderr:2023-09-22T13:36:30.468+0000 7f3202312700 -1 osd.2 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:36:30.470 INFO:tasks.ceph.osd.6.smithi174.stderr:2023-09-22T13:36:30.469+0000 7fb39f5de700 -1 osd.6 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:36:30.470 INFO:tasks.ceph.osd.7.smithi174.stderr:2023-09-22T13:36:30.469+0000 7fbcf43a5700 -1 osd.7 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:36:30.471 INFO:tasks.ceph.osd.5.smithi174.stderr:2023-09-22T13:36:30.470+0000 7f5609248700 -1 osd.5 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:36:30.471 INFO:tasks.ceph.osd.4.smithi174.stderr:2023-09-22T13:36:30.470+0000 7fe0198a9700 -1 osd.4 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:36:30.472 INFO:tasks.ceph.osd.0.smithi069.stderr:2023-09-22T13:36:30.472+0000 7f70da90c700 -1 osd.0 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:36:30.475 INFO:tasks.ceph.osd.1.smithi069.stderr:2023-09-22T13:36:30.475+0000 7fa594f73700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:36:30.476 INFO:tasks.ceph.osd.3.smithi069.stderr:2023-09-22T13:36:30.475+0000 7fb09812d700 -1 osd.3 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:36:33.257 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-22T13:36:33.555 INFO:teuthology.misc.health.smithi069.stdout: 2023-09-22T13:36:33.556 INFO:teuthology.misc.health.smithi069.stdout:{"epoch":15,"fsid":"328c464b-5dfe-4280-b09b-dba594b52bd4","created":"2023-09-22T13:36:22.739115+0000","modified":"2023-09-22T13:36:33.482022+0000","last_up_change":"2023-09-22T13:36:31.465089+0000","last_in_change":"2023-09-22T13:36:25.164335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-22T13:36:31.478363+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"0fcbd366-81e0-48ab-855b-9eee6e25c05f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6800","nonce":145165},{"type":"v1","addr":"172.21.15.69:6801","nonce":145165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6802","nonce":145165},{"type":"v1","addr":"172.21.15.69:6803","nonce":145165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6806","nonce":145165},{"type":"v1","addr":"172.21.15.69:6807","nonce":145165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6804","nonce":145165},{"type":"v1","addr":"172.21.15.69:6805","nonce":145165}]},"public_addr":"172.21.15.69:6801/145165","cluster_addr":"172.21.15.69:6803/145165","heartbeat_back_addr":"172.21.15.69:6807/145165","heartbeat_front_addr":"172.21.15.69:6805/145165","state":["exists","up"]},{"osd":1,"uuid":"8277992f-0f9d-4f92-a61a-c9d4e8f1731b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6816","nonce":145181},{"type":"v1","addr":"172.21.15.69:6817","nonce":145181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6818","nonce":145181},{"type":"v1","addr":"172.21.15.69:6819","nonce":145181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6822","nonce":145181},{"type":"v1","addr":"172.21.15.69:6823","nonce":145181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6820","nonce":145181},{"type":"v1","addr":"172.21.15.69:6821","nonce":145181}]},"public_addr":"172.21.15.69:6817/145181","cluster_addr":"172.21.15.69:6819/145181","heartbeat_back_addr":"172.21.15.69:6823/145181","heartbeat_front_addr":"172.21.15.69:6821/145181","state":["exists","up"]},{"osd":2,"uuid":"78664a54-7349-4f5a-a1df-eaa61aee8b6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6824","nonce":145182},{"type":"v1","addr":"172.21.15.69:6825","nonce":145182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6826","nonce":145182},{"type":"v1","addr":"172.21.15.69:6827","nonce":145182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6830","nonce":145182},{"type":"v1","addr":"172.21.15.69:6831","nonce":145182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6828","nonce":145182},{"type":"v1","addr":"172.21.15.69:6829","nonce":145182}]},"public_addr":"172.21.15.69:6825/145182","cluster_addr":"172.21.15.69:6827/145182","heartbeat_back_addr":"172.21.15.69:6831/145182","heartbeat_front_addr":"172.21.15.69:6829/145182","state":["exists","up"]},{"osd":3,"uuid":"b38b9327-c129-462d-b5e4-79ba3e2b0118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6808","nonce":145166},{"type":"v1","addr":"172.21.15.69:6809","nonce":145166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6810","nonce":145166},{"type":"v1","addr":"172.21.15.69:6811","nonce":145166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6814","nonce":145166},{"type":"v1","addr":"172.21.15.69:6815","nonce":145166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6812","nonce":145166},{"type":"v1","addr":"172.21.15.69:6813","nonce":145166}]},"public_addr":"172.21.15.69:6809/145166","cluster_addr":"172.21.15.69:6811/145166","heartbeat_back_addr":"172.21.15.69:6815/145166","heartbeat_front_addr":"172.21.15.69:6813/145166","state":["exists","up"]},{"osd":4,"uuid":"7fffaf9b-1a75-43bd-8a39-1bc2efba2208","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6800","nonce":145142},{"type":"v1","addr":"172.21.15.174:6801","nonce":145142}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":145142},{"type":"v1","addr":"172.21.15.174:6803","nonce":145142}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":145142},{"type":"v1","addr":"172.21.15.174:6807","nonce":145142}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":145142},{"type":"v1","addr":"172.21.15.174:6805","nonce":145142}]},"public_addr":"172.21.15.174:6801/145142","cluster_addr":"172.21.15.174:6803/145142","heartbeat_back_addr":"172.21.15.174:6807/145142","heartbeat_front_addr":"172.21.15.174:6805/145142","state":["exists","up"]},{"osd":5,"uuid":"3d45afee-adf7-43fb-912f-ca92d46d2758","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":145141},{"type":"v1","addr":"172.21.15.174:6809","nonce":145141}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":145141},{"type":"v1","addr":"172.21.15.174:6811","nonce":145141}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":145141},{"type":"v1","addr":"172.21.15.174:6815","nonce":145141}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":145141},{"type":"v1","addr":"172.21.15.174:6813","nonce":145141}]},"public_addr":"172.21.15.174:6809/145141","cluster_addr":"172.21.15.174:6811/145141","heartbeat_back_addr":"172.21.15.174:6815/145141","heartbeat_front_addr":"172.21.15.174:6813/145141","state":["exists","up"]},{"osd":6,"uuid":"3c517ba1-d327-487c-8c36-7f78528cba1d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":145172},{"type":"v1","addr":"172.21.15.174:6825","nonce":145172}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":145172},{"type":"v1","addr":"172.21.15.174:6827","nonce":145172}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":145172},{"type":"v1","addr":"172.21.15.174:6831","nonce":145172}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":145172},{"type":"v1","addr":"172.21.15.174:6829","nonce":145172}]},"public_addr":"172.21.15.174:6825/145172","cluster_addr":"172.21.15.174:6827/145172","heartbeat_back_addr":"172.21.15.174:6831/145172","heartbeat_front_addr":"172.21.15.174:6829/145172","state":["exists","up"]},{"osd":7,"uuid":"3056e758-0438-4e6e-990b-f972f8a648f9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":145171},{"type":"v1","addr":"172.21.15.174:6817","nonce":145171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":145171},{"type":"v1","addr":"172.21.15.174:6819","nonce":145171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":145171},{"type":"v1","addr":"172.21.15.174:6823","nonce":145171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":145171},{"type":"v1","addr":"172.21.15.174:6821","nonce":145171}]},"public_addr":"172.21.15.174:6817/145171","cluster_addr":"172.21.15.174:6819/145171","heartbeat_back_addr":"172.21.15.174:6823/145171","heartbeat_front_addr":"172.21.15.174:6821/145171","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.349610+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.372437+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.352193+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.283103+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.226388+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.298870+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.342107+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.319830+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:36:33.568 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-22T13:36:33.568 INFO:tasks.ceph:Creating RBD pool 2023-09-22T13:36:33.568 DEBUG:teuthology.orchestra.run.smithi069:> sudo ceph --cluster ceph osd pool create rbd 8 2023-09-22T13:36:34.498 INFO:teuthology.orchestra.run.smithi069.stderr:pool 'rbd' created 2023-09-22T13:36:34.512 DEBUG:teuthology.orchestra.run.smithi069:> sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-22T13:36:35.501 INFO:teuthology.orchestra.run.smithi069.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-22T13:36:35.516 INFO:tasks.ceph:Starting mds daemons in cluster ceph... 2023-09-22T13:36:35.517 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-22T13:36:35.517 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2023-09-22T13:36:35.517 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2023-09-22T13:36:35.517 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2023-09-22T13:36:35.861 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:36:35.873 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":4,"active_gid":4105,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6832","nonce":144867},{"type":"v1","addr":"172.21.15.69:6833","nonce":144867}]},"active_addr":"172.21.15.69:6833/144867","active_change":"2023-09-22T13:36:27.445151+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":4102,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.69:0","nonce":1168695615}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.69:0","nonce":221955698}]}]} 2023-09-22T13:36:35.877 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2023-09-22T13:36:35.877 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2023-09-22T13:36:35.877 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-09-22T13:36:36.165 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:36:36.165 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":17,"fsid":"328c464b-5dfe-4280-b09b-dba594b52bd4","created":"2023-09-22T13:36:22.739115+0000","modified":"2023-09-22T13:36:35.491153+0000","last_up_change":"2023-09-22T13:36:31.465089+0000","last_in_change":"2023-09-22T13:36:25.164335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-22T13:36:31.478363+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-22T13:36:33.855627+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"0fcbd366-81e0-48ab-855b-9eee6e25c05f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6800","nonce":145165},{"type":"v1","addr":"172.21.15.69:6801","nonce":145165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6802","nonce":145165},{"type":"v1","addr":"172.21.15.69:6803","nonce":145165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6806","nonce":145165},{"type":"v1","addr":"172.21.15.69:6807","nonce":145165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6804","nonce":145165},{"type":"v1","addr":"172.21.15.69:6805","nonce":145165}]},"public_addr":"172.21.15.69:6801/145165","cluster_addr":"172.21.15.69:6803/145165","heartbeat_back_addr":"172.21.15.69:6807/145165","heartbeat_front_addr":"172.21.15.69:6805/145165","state":["exists","up"]},{"osd":1,"uuid":"8277992f-0f9d-4f92-a61a-c9d4e8f1731b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6816","nonce":145181},{"type":"v1","addr":"172.21.15.69:6817","nonce":145181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6818","nonce":145181},{"type":"v1","addr":"172.21.15.69:6819","nonce":145181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6822","nonce":145181},{"type":"v1","addr":"172.21.15.69:6823","nonce":145181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6820","nonce":145181},{"type":"v1","addr":"172.21.15.69:6821","nonce":145181}]},"public_addr":"172.21.15.69:6817/145181","cluster_addr":"172.21.15.69:6819/145181","heartbeat_back_addr":"172.21.15.69:6823/145181","heartbeat_front_addr":"172.21.15.69:6821/145181","state":["exists","up"]},{"osd":2,"uuid":"78664a54-7349-4f5a-a1df-eaa61aee8b6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6824","nonce":145182},{"type":"v1","addr":"172.21.15.69:6825","nonce":145182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6826","nonce":145182},{"type":"v1","addr":"172.21.15.69:6827","nonce":145182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6830","nonce":145182},{"type":"v1","addr":"172.21.15.69:6831","nonce":145182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6828","nonce":145182},{"type":"v1","addr":"172.21.15.69:6829","nonce":145182}]},"public_addr":"172.21.15.69:6825/145182","cluster_addr":"172.21.15.69:6827/145182","heartbeat_back_addr":"172.21.15.69:6831/145182","heartbeat_front_addr":"172.21.15.69:6829/145182","state":["exists","up"]},{"osd":3,"uuid":"b38b9327-c129-462d-b5e4-79ba3e2b0118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6808","nonce":145166},{"type":"v1","addr":"172.21.15.69:6809","nonce":145166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6810","nonce":145166},{"type":"v1","addr":"172.21.15.69:6811","nonce":145166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6814","nonce":145166},{"type":"v1","addr":"172.21.15.69:6815","nonce":145166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6812","nonce":145166},{"type":"v1","addr":"172.21.15.69:6813","nonce":145166}]},"public_addr":"172.21.15.69:6809/145166","cluster_addr":"172.21.15.69:6811/145166","heartbeat_back_addr":"172.21.15.69:6815/145166","heartbeat_front_addr":"172.21.15.69:6813/145166","state":["exists","up"]},{"osd":4,"uuid":"7fffaf9b-1a75-43bd-8a39-1bc2efba2208","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6800","nonce":145142},{"type":"v1","addr":"172.21.15.174:6801","nonce":145142}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":145142},{"type":"v1","addr":"172.21.15.174:6803","nonce":145142}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":145142},{"type":"v1","addr":"172.21.15.174:6807","nonce":145142}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":145142},{"type":"v1","addr":"172.21.15.174:6805","nonce":145142}]},"public_addr":"172.21.15.174:6801/145142","cluster_addr":"172.21.15.174:6803/145142","heartbeat_back_addr":"172.21.15.174:6807/145142","heartbeat_front_addr":"172.21.15.174:6805/145142","state":["exists","up"]},{"osd":5,"uuid":"3d45afee-adf7-43fb-912f-ca92d46d2758","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":145141},{"type":"v1","addr":"172.21.15.174:6809","nonce":145141}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":145141},{"type":"v1","addr":"172.21.15.174:6811","nonce":145141}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":145141},{"type":"v1","addr":"172.21.15.174:6815","nonce":145141}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":145141},{"type":"v1","addr":"172.21.15.174:6813","nonce":145141}]},"public_addr":"172.21.15.174:6809/145141","cluster_addr":"172.21.15.174:6811/145141","heartbeat_back_addr":"172.21.15.174:6815/145141","heartbeat_front_addr":"172.21.15.174:6813/145141","state":["exists","up"]},{"osd":6,"uuid":"3c517ba1-d327-487c-8c36-7f78528cba1d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":145172},{"type":"v1","addr":"172.21.15.174:6825","nonce":145172}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":145172},{"type":"v1","addr":"172.21.15.174:6827","nonce":145172}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":145172},{"type":"v1","addr":"172.21.15.174:6831","nonce":145172}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":145172},{"type":"v1","addr":"172.21.15.174:6829","nonce":145172}]},"public_addr":"172.21.15.174:6825/145172","cluster_addr":"172.21.15.174:6827/145172","heartbeat_back_addr":"172.21.15.174:6831/145172","heartbeat_front_addr":"172.21.15.174:6829/145172","state":["exists","up"]},{"osd":7,"uuid":"3056e758-0438-4e6e-990b-f972f8a648f9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":145171},{"type":"v1","addr":"172.21.15.174:6817","nonce":145171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":145171},{"type":"v1","addr":"172.21.15.174:6819","nonce":145171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":145171},{"type":"v1","addr":"172.21.15.174:6823","nonce":145171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":145171},{"type":"v1","addr":"172.21.15.174:6821","nonce":145171}]},"public_addr":"172.21.15.174:6817/145171","cluster_addr":"172.21.15.174:6819/145171","heartbeat_back_addr":"172.21.15.174:6823/145171","heartbeat_front_addr":"172.21.15.174:6821/145171","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.349610+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.372437+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.352193+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.283103+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.226388+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.298870+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.342107+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.319830+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:36:36.179 INFO:tasks.ceph.ceph_manager.ceph:all up! 2023-09-22T13:36:36.179 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-09-22T13:36:36.469 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:36:36.469 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":17,"fsid":"328c464b-5dfe-4280-b09b-dba594b52bd4","created":"2023-09-22T13:36:22.739115+0000","modified":"2023-09-22T13:36:35.491153+0000","last_up_change":"2023-09-22T13:36:31.465089+0000","last_in_change":"2023-09-22T13:36:25.164335+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-22T13:36:31.478363+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-22T13:36:33.855627+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"0fcbd366-81e0-48ab-855b-9eee6e25c05f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6800","nonce":145165},{"type":"v1","addr":"172.21.15.69:6801","nonce":145165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6802","nonce":145165},{"type":"v1","addr":"172.21.15.69:6803","nonce":145165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6806","nonce":145165},{"type":"v1","addr":"172.21.15.69:6807","nonce":145165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6804","nonce":145165},{"type":"v1","addr":"172.21.15.69:6805","nonce":145165}]},"public_addr":"172.21.15.69:6801/145165","cluster_addr":"172.21.15.69:6803/145165","heartbeat_back_addr":"172.21.15.69:6807/145165","heartbeat_front_addr":"172.21.15.69:6805/145165","state":["exists","up"]},{"osd":1,"uuid":"8277992f-0f9d-4f92-a61a-c9d4e8f1731b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6816","nonce":145181},{"type":"v1","addr":"172.21.15.69:6817","nonce":145181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6818","nonce":145181},{"type":"v1","addr":"172.21.15.69:6819","nonce":145181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6822","nonce":145181},{"type":"v1","addr":"172.21.15.69:6823","nonce":145181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6820","nonce":145181},{"type":"v1","addr":"172.21.15.69:6821","nonce":145181}]},"public_addr":"172.21.15.69:6817/145181","cluster_addr":"172.21.15.69:6819/145181","heartbeat_back_addr":"172.21.15.69:6823/145181","heartbeat_front_addr":"172.21.15.69:6821/145181","state":["exists","up"]},{"osd":2,"uuid":"78664a54-7349-4f5a-a1df-eaa61aee8b6a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6824","nonce":145182},{"type":"v1","addr":"172.21.15.69:6825","nonce":145182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6826","nonce":145182},{"type":"v1","addr":"172.21.15.69:6827","nonce":145182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6830","nonce":145182},{"type":"v1","addr":"172.21.15.69:6831","nonce":145182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6828","nonce":145182},{"type":"v1","addr":"172.21.15.69:6829","nonce":145182}]},"public_addr":"172.21.15.69:6825/145182","cluster_addr":"172.21.15.69:6827/145182","heartbeat_back_addr":"172.21.15.69:6831/145182","heartbeat_front_addr":"172.21.15.69:6829/145182","state":["exists","up"]},{"osd":3,"uuid":"b38b9327-c129-462d-b5e4-79ba3e2b0118","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6808","nonce":145166},{"type":"v1","addr":"172.21.15.69:6809","nonce":145166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6810","nonce":145166},{"type":"v1","addr":"172.21.15.69:6811","nonce":145166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6814","nonce":145166},{"type":"v1","addr":"172.21.15.69:6815","nonce":145166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6812","nonce":145166},{"type":"v1","addr":"172.21.15.69:6813","nonce":145166}]},"public_addr":"172.21.15.69:6809/145166","cluster_addr":"172.21.15.69:6811/145166","heartbeat_back_addr":"172.21.15.69:6815/145166","heartbeat_front_addr":"172.21.15.69:6813/145166","state":["exists","up"]},{"osd":4,"uuid":"7fffaf9b-1a75-43bd-8a39-1bc2efba2208","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6800","nonce":145142},{"type":"v1","addr":"172.21.15.174:6801","nonce":145142}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":145142},{"type":"v1","addr":"172.21.15.174:6803","nonce":145142}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":145142},{"type":"v1","addr":"172.21.15.174:6807","nonce":145142}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":145142},{"type":"v1","addr":"172.21.15.174:6805","nonce":145142}]},"public_addr":"172.21.15.174:6801/145142","cluster_addr":"172.21.15.174:6803/145142","heartbeat_back_addr":"172.21.15.174:6807/145142","heartbeat_front_addr":"172.21.15.174:6805/145142","state":["exists","up"]},{"osd":5,"uuid":"3d45afee-adf7-43fb-912f-ca92d46d2758","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":145141},{"type":"v1","addr":"172.21.15.174:6809","nonce":145141}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":145141},{"type":"v1","addr":"172.21.15.174:6811","nonce":145141}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":145141},{"type":"v1","addr":"172.21.15.174:6815","nonce":145141}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":145141},{"type":"v1","addr":"172.21.15.174:6813","nonce":145141}]},"public_addr":"172.21.15.174:6809/145141","cluster_addr":"172.21.15.174:6811/145141","heartbeat_back_addr":"172.21.15.174:6815/145141","heartbeat_front_addr":"172.21.15.174:6813/145141","state":["exists","up"]},{"osd":6,"uuid":"3c517ba1-d327-487c-8c36-7f78528cba1d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":145172},{"type":"v1","addr":"172.21.15.174:6825","nonce":145172}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":145172},{"type":"v1","addr":"172.21.15.174:6827","nonce":145172}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":145172},{"type":"v1","addr":"172.21.15.174:6831","nonce":145172}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":145172},{"type":"v1","addr":"172.21.15.174:6829","nonce":145172}]},"public_addr":"172.21.15.174:6825/145172","cluster_addr":"172.21.15.174:6827/145172","heartbeat_back_addr":"172.21.15.174:6831/145172","heartbeat_front_addr":"172.21.15.174:6829/145172","state":["exists","up"]},{"osd":7,"uuid":"3056e758-0438-4e6e-990b-f972f8a648f9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":145171},{"type":"v1","addr":"172.21.15.174:6817","nonce":145171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":145171},{"type":"v1","addr":"172.21.15.174:6819","nonce":145171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":145171},{"type":"v1","addr":"172.21.15.174:6823","nonce":145171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":145171},{"type":"v1","addr":"172.21.15.174:6821","nonce":145171}]},"public_addr":"172.21.15.174:6817/145171","cluster_addr":"172.21.15.174:6819/145171","heartbeat_back_addr":"172.21.15.174:6823/145171","heartbeat_front_addr":"172.21.15.174:6821/145171","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.349610+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.372437+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.352193+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.283103+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.226388+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.298870+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.342107+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:36:30.319830+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:36:36.483 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2023-09-22T13:36:36.634 INFO:teuthology.orchestra.run.smithi069.stdout:55834574850 2023-09-22T13:36:36.634 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2023-09-22T13:36:36.784 INFO:teuthology.orchestra.run.smithi069.stdout:55834574850 2023-09-22T13:36:36.784 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2023-09-22T13:36:36.932 INFO:teuthology.orchestra.run.smithi069.stdout:55834574850 2023-09-22T13:36:36.933 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2023-09-22T13:36:37.081 INFO:teuthology.orchestra.run.smithi069.stdout:55834574851 2023-09-22T13:36:37.081 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.4 flush_pg_stats 2023-09-22T13:36:37.230 INFO:teuthology.orchestra.run.smithi069.stdout:55834574852 2023-09-22T13:36:37.230 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.5 flush_pg_stats 2023-09-22T13:36:37.379 INFO:teuthology.orchestra.run.smithi069.stdout:55834574850 2023-09-22T13:36:37.379 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.6 flush_pg_stats 2023-09-22T13:36:37.532 INFO:teuthology.orchestra.run.smithi069.stdout:55834574852 2023-09-22T13:36:37.532 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.7 flush_pg_stats 2023-09-22T13:36:37.682 INFO:teuthology.orchestra.run.smithi069.stdout:55834574851 2023-09-22T13:36:37.683 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2023-09-22T13:36:37.970 INFO:teuthology.orchestra.run.smithi069.stdout:55834574850 2023-09-22T13:36:37.984 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574850 got 55834574850 for osd.0 2023-09-22T13:36:37.984 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2023-09-22T13:36:38.271 INFO:teuthology.orchestra.run.smithi069.stdout:55834574850 2023-09-22T13:36:38.284 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574850 got 55834574850 for osd.1 2023-09-22T13:36:38.284 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2023-09-22T13:36:38.578 INFO:teuthology.orchestra.run.smithi069.stdout:55834574850 2023-09-22T13:36:38.591 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574850 got 55834574850 for osd.2 2023-09-22T13:36:38.592 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2023-09-22T13:36:38.878 INFO:teuthology.orchestra.run.smithi069.stdout:55834574851 2023-09-22T13:36:38.891 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574851 got 55834574851 for osd.3 2023-09-22T13:36:38.892 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.4 2023-09-22T13:36:39.180 INFO:teuthology.orchestra.run.smithi069.stdout:55834574852 2023-09-22T13:36:39.194 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574852 got 55834574852 for osd.4 2023-09-22T13:36:39.194 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.5 2023-09-22T13:36:39.484 INFO:teuthology.orchestra.run.smithi069.stdout:55834574850 2023-09-22T13:36:39.497 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574850 got 55834574850 for osd.5 2023-09-22T13:36:39.498 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.6 2023-09-22T13:36:39.791 INFO:teuthology.orchestra.run.smithi069.stdout:55834574852 2023-09-22T13:36:39.805 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574852 got 55834574852 for osd.6 2023-09-22T13:36:39.805 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.7 2023-09-22T13:36:40.091 INFO:teuthology.orchestra.run.smithi069.stdout:55834574851 2023-09-22T13:36:40.105 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574851 got 55834574851 for osd.7 2023-09-22T13:36:40.105 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2023-09-22T13:36:40.105 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2023-09-22T13:36:40.391 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:36:40.393 INFO:teuthology.orchestra.run.smithi069.stderr:dumped all 2023-09-22T13:36:40.403 INFO:teuthology.orchestra.run.smithi069.stdout:{"pg_ready":true,"pg_map":{"version":15,"stamp":"2023-09-22T13:36:39.451859+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":7,"kb":754974720,"kb_used":2378176,"kb_used_data":1408,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":752596544,"statfs":{"total":773094113280,"available":770658861056,"internally_reserved":0,"allocated":1441792,"data_stored":468464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.509685+0000","last_change":"2023-09-22T13:36:35.505632+0000","last_active":"2023-09-22T13:36:35.509685+0000","last_peered":"2023-09-22T13:36:35.509685+0000","last_clean":"2023-09-22T13:36:35.509685+0000","last_became_active":"2023-09-22T13:36:35.505325+0000","last_became_peered":"2023-09-22T13:36:35.505325+0000","last_unstale":"2023-09-22T13:36:35.509685+0000","last_undegraded":"2023-09-22T13:36:35.509685+0000","last_fullsized":"2023-09-22T13:36:35.509685+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.507675+0000","last_change":"2023-09-22T13:36:35.505990+0000","last_active":"2023-09-22T13:36:35.507675+0000","last_peered":"2023-09-22T13:36:35.507675+0000","last_clean":"2023-09-22T13:36:35.507675+0000","last_became_active":"2023-09-22T13:36:35.505698+0000","last_became_peered":"2023-09-22T13:36:35.505698+0000","last_unstale":"2023-09-22T13:36:35.507675+0000","last_undegraded":"2023-09-22T13:36:35.507675+0000","last_fullsized":"2023-09-22T13:36:35.507675+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.654468+0000","last_change":"2023-09-22T13:36:35.652690+0000","last_active":"2023-09-22T13:36:35.654468+0000","last_peered":"2023-09-22T13:36:35.654468+0000","last_clean":"2023-09-22T13:36:35.654468+0000","last_became_active":"2023-09-22T13:36:35.652371+0000","last_became_peered":"2023-09-22T13:36:35.652371+0000","last_unstale":"2023-09-22T13:36:35.654468+0000","last_undegraded":"2023-09-22T13:36:35.654468+0000","last_fullsized":"2023-09-22T13:36:35.654468+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.507061+0000","last_change":"2023-09-22T13:36:35.501983+0000","last_active":"2023-09-22T13:36:35.507061+0000","last_peered":"2023-09-22T13:36:35.507061+0000","last_clean":"2023-09-22T13:36:35.507061+0000","last_became_active":"2023-09-22T13:36:35.501678+0000","last_became_peered":"2023-09-22T13:36:35.501678+0000","last_unstale":"2023-09-22T13:36:35.507061+0000","last_undegraded":"2023-09-22T13:36:35.507061+0000","last_fullsized":"2023-09-22T13:36:35.507061+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":11,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.499885+0000","last_change":"2023-09-22T13:36:33.491427+0000","last_active":"2023-09-22T13:36:35.499885+0000","last_peered":"2023-09-22T13:36:35.499885+0000","last_clean":"2023-09-22T13:36:35.499885+0000","last_became_active":"2023-09-22T13:36:33.491114+0000","last_became_peered":"2023-09-22T13:36:33.491114+0000","last_unstale":"2023-09-22T13:36:35.499885+0000","last_undegraded":"2023-09-22T13:36:35.499885+0000","last_fullsized":"2023-09-22T13:36:35.499885+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:32.472338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:32.472338+0000","last_clean_scrub_stamp":"2023-09-22T13:36:32.472338+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.511386+0000","last_change":"2023-09-22T13:36:35.506093+0000","last_active":"2023-09-22T13:36:35.511386+0000","last_peered":"2023-09-22T13:36:35.511386+0000","last_clean":"2023-09-22T13:36:35.511386+0000","last_became_active":"2023-09-22T13:36:35.505735+0000","last_became_peered":"2023-09-22T13:36:35.505735+0000","last_unstale":"2023-09-22T13:36:35.511386+0000","last_undegraded":"2023-09-22T13:36:35.511386+0000","last_fullsized":"2023-09-22T13:36:35.511386+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.499512+0000","last_change":"2023-09-22T13:36:35.498625+0000","last_active":"2023-09-22T13:36:35.499512+0000","last_peered":"2023-09-22T13:36:35.499512+0000","last_clean":"2023-09-22T13:36:35.499512+0000","last_became_active":"2023-09-22T13:36:35.498344+0000","last_became_peered":"2023-09-22T13:36:35.498344+0000","last_unstale":"2023-09-22T13:36:35.499512+0000","last_undegraded":"2023-09-22T13:36:35.499512+0000","last_fullsized":"2023-09-22T13:36:35.499512+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.511368+0000","last_change":"2023-09-22T13:36:35.505682+0000","last_active":"2023-09-22T13:36:35.511368+0000","last_peered":"2023-09-22T13:36:35.511368+0000","last_clean":"2023-09-22T13:36:35.511368+0000","last_became_active":"2023-09-22T13:36:35.505323+0000","last_became_peered":"2023-09-22T13:36:35.505323+0000","last_unstale":"2023-09-22T13:36:35.511368+0000","last_undegraded":"2023-09-22T13:36:35.511368+0000","last_fullsized":"2023-09-22T13:36:35.511368+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.653121+0000","last_change":"2023-09-22T13:36:35.652187+0000","last_active":"2023-09-22T13:36:35.653121+0000","last_peered":"2023-09-22T13:36:35.653121+0000","last_clean":"2023-09-22T13:36:35.653121+0000","last_became_active":"2023-09-22T13:36:35.651923+0000","last_became_peered":"2023-09-22T13:36:35.651923+0000","last_unstale":"2023-09-22T13:36:35.653121+0000","last_undegraded":"2023-09-22T13:36:35.653121+0000","last_fullsized":"2023-09-22T13:36:35.653121+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574851,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44500000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59899999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47799999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66000000000000003}]}]},{"osd":6,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.377}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47599999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46800000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574851,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.245}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27600000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33300000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61699999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67400000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64300000000000002}]}]},{"osd":0,"up_from":13,"seq":55834574851,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33700000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26600000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67000000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68899999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59299999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64000000000000001}]}]},{"osd":2,"up_from":13,"seq":55834574851,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44900000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40899999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52500000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72499999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53400000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]}]},{"osd":3,"up_from":13,"seq":55834574852,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28799999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38600000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46000000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63500000000000001}]}]},{"osd":4,"up_from":13,"seq":55834574853,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58099999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65700000000000003}]}]},{"osd":5,"up_from":13,"seq":55834574851,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54100000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41199999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.495}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58799999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57299999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-22T13:36:40.405 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2023-09-22T13:36:40.691 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:36:40.693 INFO:teuthology.orchestra.run.smithi069.stderr:dumped all 2023-09-22T13:36:40.704 INFO:teuthology.orchestra.run.smithi069.stdout:{"pg_ready":true,"pg_map":{"version":15,"stamp":"2023-09-22T13:36:39.451859+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":7,"kb":754974720,"kb_used":2378176,"kb_used_data":1408,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":752596544,"statfs":{"total":773094113280,"available":770658861056,"internally_reserved":0,"allocated":1441792,"data_stored":468464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.509685+0000","last_change":"2023-09-22T13:36:35.505632+0000","last_active":"2023-09-22T13:36:35.509685+0000","last_peered":"2023-09-22T13:36:35.509685+0000","last_clean":"2023-09-22T13:36:35.509685+0000","last_became_active":"2023-09-22T13:36:35.505325+0000","last_became_peered":"2023-09-22T13:36:35.505325+0000","last_unstale":"2023-09-22T13:36:35.509685+0000","last_undegraded":"2023-09-22T13:36:35.509685+0000","last_fullsized":"2023-09-22T13:36:35.509685+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.507675+0000","last_change":"2023-09-22T13:36:35.505990+0000","last_active":"2023-09-22T13:36:35.507675+0000","last_peered":"2023-09-22T13:36:35.507675+0000","last_clean":"2023-09-22T13:36:35.507675+0000","last_became_active":"2023-09-22T13:36:35.505698+0000","last_became_peered":"2023-09-22T13:36:35.505698+0000","last_unstale":"2023-09-22T13:36:35.507675+0000","last_undegraded":"2023-09-22T13:36:35.507675+0000","last_fullsized":"2023-09-22T13:36:35.507675+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.654468+0000","last_change":"2023-09-22T13:36:35.652690+0000","last_active":"2023-09-22T13:36:35.654468+0000","last_peered":"2023-09-22T13:36:35.654468+0000","last_clean":"2023-09-22T13:36:35.654468+0000","last_became_active":"2023-09-22T13:36:35.652371+0000","last_became_peered":"2023-09-22T13:36:35.652371+0000","last_unstale":"2023-09-22T13:36:35.654468+0000","last_undegraded":"2023-09-22T13:36:35.654468+0000","last_fullsized":"2023-09-22T13:36:35.654468+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.507061+0000","last_change":"2023-09-22T13:36:35.501983+0000","last_active":"2023-09-22T13:36:35.507061+0000","last_peered":"2023-09-22T13:36:35.507061+0000","last_clean":"2023-09-22T13:36:35.507061+0000","last_became_active":"2023-09-22T13:36:35.501678+0000","last_became_peered":"2023-09-22T13:36:35.501678+0000","last_unstale":"2023-09-22T13:36:35.507061+0000","last_undegraded":"2023-09-22T13:36:35.507061+0000","last_fullsized":"2023-09-22T13:36:35.507061+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":11,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.499885+0000","last_change":"2023-09-22T13:36:33.491427+0000","last_active":"2023-09-22T13:36:35.499885+0000","last_peered":"2023-09-22T13:36:35.499885+0000","last_clean":"2023-09-22T13:36:35.499885+0000","last_became_active":"2023-09-22T13:36:33.491114+0000","last_became_peered":"2023-09-22T13:36:33.491114+0000","last_unstale":"2023-09-22T13:36:35.499885+0000","last_undegraded":"2023-09-22T13:36:35.499885+0000","last_fullsized":"2023-09-22T13:36:35.499885+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:32.472338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:32.472338+0000","last_clean_scrub_stamp":"2023-09-22T13:36:32.472338+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.511386+0000","last_change":"2023-09-22T13:36:35.506093+0000","last_active":"2023-09-22T13:36:35.511386+0000","last_peered":"2023-09-22T13:36:35.511386+0000","last_clean":"2023-09-22T13:36:35.511386+0000","last_became_active":"2023-09-22T13:36:35.505735+0000","last_became_peered":"2023-09-22T13:36:35.505735+0000","last_unstale":"2023-09-22T13:36:35.511386+0000","last_undegraded":"2023-09-22T13:36:35.511386+0000","last_fullsized":"2023-09-22T13:36:35.511386+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.499512+0000","last_change":"2023-09-22T13:36:35.498625+0000","last_active":"2023-09-22T13:36:35.499512+0000","last_peered":"2023-09-22T13:36:35.499512+0000","last_clean":"2023-09-22T13:36:35.499512+0000","last_became_active":"2023-09-22T13:36:35.498344+0000","last_became_peered":"2023-09-22T13:36:35.498344+0000","last_unstale":"2023-09-22T13:36:35.499512+0000","last_undegraded":"2023-09-22T13:36:35.499512+0000","last_fullsized":"2023-09-22T13:36:35.499512+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.511368+0000","last_change":"2023-09-22T13:36:35.505682+0000","last_active":"2023-09-22T13:36:35.511368+0000","last_peered":"2023-09-22T13:36:35.511368+0000","last_clean":"2023-09-22T13:36:35.511368+0000","last_became_active":"2023-09-22T13:36:35.505323+0000","last_became_peered":"2023-09-22T13:36:35.505323+0000","last_unstale":"2023-09-22T13:36:35.511368+0000","last_undegraded":"2023-09-22T13:36:35.511368+0000","last_fullsized":"2023-09-22T13:36:35.511368+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:36:35.653121+0000","last_change":"2023-09-22T13:36:35.652187+0000","last_active":"2023-09-22T13:36:35.653121+0000","last_peered":"2023-09-22T13:36:35.653121+0000","last_clean":"2023-09-22T13:36:35.653121+0000","last_became_active":"2023-09-22T13:36:35.651923+0000","last_became_peered":"2023-09-22T13:36:35.651923+0000","last_unstale":"2023-09-22T13:36:35.653121+0000","last_undegraded":"2023-09-22T13:36:35.653121+0000","last_fullsized":"2023-09-22T13:36:35.653121+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:36:34.489816+0000","last_clean_scrub_stamp":"2023-09-22T13:36:34.489816+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574851,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44500000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59899999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47799999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66000000000000003}]}]},{"osd":6,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.377}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47599999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46800000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574851,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.245}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27600000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33300000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61699999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67400000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64300000000000002}]}]},{"osd":0,"up_from":13,"seq":55834574851,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33700000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26600000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67000000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68899999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59299999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64000000000000001}]}]},{"osd":2,"up_from":13,"seq":55834574851,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44900000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40899999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52500000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72499999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53400000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]}]},{"osd":3,"up_from":13,"seq":55834574852,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28799999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38600000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46000000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63500000000000001}]}]},{"osd":4,"up_from":13,"seq":55834574853,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58099999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65700000000000003}]}]},{"osd":5,"up_from":13,"seq":55834574851,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297272,"kb_used_data":176,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074568,"statfs":{"total":96636764160,"available":96332357632,"internally_reserved":0,"allocated":180224,"data_stored":58558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54100000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41199999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.495}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58799999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57299999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-22T13:36:40.705 INFO:tasks.ceph.ceph_manager.ceph:clean! 2023-09-22T13:36:40.706 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-22T13:36:40.706 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2023-09-22T13:36:40.706 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2023-09-22T13:36:41.027 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:36:41.040 INFO:teuthology.orchestra.run.smithi069.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-22T13:36:41.040 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2023-09-22T13:36:41.041 INFO:teuthology.run_tasks:Running task openssl_keys... 2023-09-22T13:36:41.056 DEBUG:teuthology.task:Applying overrides for task openssl_keys: {'rgw.client.0': {'ca': 'root', 'client': 'client.0', 'embed-key': True}, 'rgw.client.1': {'ca': 'root', 'client': 'client.1', 'embed-key': True}, 'root': {'client': 'client.0', 'cn': 'teuthology', 'install': ['client.0', 'client.1'], 'key-type': 'rsa:4096'}} 2023-09-22T13:36:41.056 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p /home/ubuntu/cephtest/ca 2023-09-22T13:36:41.074 DEBUG:teuthology.orchestra.run.smithi069:> openssl req -x509 -nodes -newkey rsa:4096 -keyout /home/ubuntu/cephtest/ca/root.key -days 365 -out /home/ubuntu/cephtest/ca/root.crt -subj /CN=teuthology 2023-09-22T13:36:41.150 INFO:teuthology.orchestra.run.smithi069.stderr:Generating a RSA private key 2023-09-22T13:36:41.223 INFO:teuthology.orchestra.run.smithi069.stderr:.....................++++ 2023-09-22T13:36:41.260 INFO:teuthology.orchestra.run.smithi069.stderr:..........++++ 2023-09-22T13:36:41.260 INFO:teuthology.orchestra.run.smithi069.stderr:writing new private key to '/home/ubuntu/cephtest/ca/root.key' 2023-09-22T13:36:41.260 INFO:teuthology.orchestra.run.smithi069.stderr:----- 2023-09-22T13:36:41.266 DEBUG:teuthology.orchestra.run.smithi069:> sudo cp /home/ubuntu/cephtest/ca/root.crt /usr/share/pki/ca-trust-source/anchors/root.crt 2023-09-22T13:36:41.343 DEBUG:teuthology.orchestra.run.smithi069:> sudo update-ca-trust 2023-09-22T13:36:42.261 DEBUG:teuthology.orchestra.run.smithi174:> mkdir -p /home/ubuntu/cephtest/ca 2023-09-22T13:36:42.283 INFO:tasks.openssl_keys:copying from ubuntu@smithi069.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/root.crt to ubuntu@smithi174.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/root.crt... 2023-09-22T13:36:42.304 DEBUG:teuthology.orchestra.remote:smithi069:/home/ubuntu/cephtest/ca/root.crt is 2KB 2023-09-22T13:36:42.377 DEBUG:teuthology.orchestra.run.smithi174:> sudo mv /home/ubuntu/cephtest/ca/root.crt /usr/share/pki/ca-trust-source/anchors/root.crt 2023-09-22T13:36:42.413 DEBUG:teuthology.orchestra.run.smithi174:> sudo update-ca-trust 2023-09-22T13:36:43.348 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p /home/ubuntu/cephtest/ca 2023-09-22T13:36:43.370 DEBUG:teuthology.orchestra.run.smithi069:> openssl req -nodes -newkey rsa:2048 -keyout /home/ubuntu/cephtest/ca/rgw.client.0.key -out /home/ubuntu/cephtest/ca/rgw.client.0.csr -subj /CN=smithi069.front.sepia.ceph.com 2023-09-22T13:36:43.431 INFO:teuthology.orchestra.run.smithi069.stderr:Generating a RSA private key 2023-09-22T13:36:43.476 INFO:teuthology.orchestra.run.smithi069.stderr:............................................+++++ 2023-09-22T13:36:43.505 INFO:teuthology.orchestra.run.smithi069.stderr:.............................+++++ 2023-09-22T13:36:43.505 INFO:teuthology.orchestra.run.smithi069.stderr:writing new private key to '/home/ubuntu/cephtest/ca/rgw.client.0.key' 2023-09-22T13:36:43.505 INFO:teuthology.orchestra.run.smithi069.stderr:----- 2023-09-22T13:36:43.507 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:43.507 DEBUG:teuthology.orchestra.run.smithi069:> dd of=/home/ubuntu/cephtest/ca/rgw.client.0.ext 2023-09-22T13:36:43.566 DEBUG:teuthology.orchestra.run.smithi069:> openssl x509 -req -in /home/ubuntu/cephtest/ca/rgw.client.0.csr -CA /home/ubuntu/cephtest/ca/root.crt -CAkey /home/ubuntu/cephtest/ca/root.key -CAcreateserial -out /home/ubuntu/cephtest/ca/rgw.client.0.crt -days 365 -sha256 -extfile /home/ubuntu/cephtest/ca/rgw.client.0.ext 2023-09-22T13:36:43.630 INFO:teuthology.orchestra.run.smithi069.stderr:Signature ok 2023-09-22T13:36:43.630 INFO:teuthology.orchestra.run.smithi069.stderr:subject=CN = smithi069.front.sepia.ceph.com 2023-09-22T13:36:43.631 INFO:teuthology.orchestra.run.smithi069.stderr:Getting CA Private Key 2023-09-22T13:36:43.636 DEBUG:teuthology.orchestra.run.smithi069:> rm /home/ubuntu/cephtest/ca/rgw.client.0.csr /home/ubuntu/cephtest/ca/root.srl /home/ubuntu/cephtest/ca/rgw.client.0.ext 2023-09-22T13:36:43.694 DEBUG:teuthology.orchestra.run.smithi069:> openssl verify -CAfile /home/ubuntu/cephtest/ca/root.crt /home/ubuntu/cephtest/ca/rgw.client.0.crt 2023-09-22T13:36:43.754 INFO:teuthology.orchestra.run.smithi069.stdout:/home/ubuntu/cephtest/ca/rgw.client.0.crt: OK 2023-09-22T13:36:43.755 DEBUG:teuthology.orchestra.run.smithi069:> cat /home/ubuntu/cephtest/ca/rgw.client.0.key >> /home/ubuntu/cephtest/ca/rgw.client.0.crt 2023-09-22T13:36:43.812 DEBUG:teuthology.orchestra.run.smithi174:> mkdir -p /home/ubuntu/cephtest/ca 2023-09-22T13:36:43.831 DEBUG:teuthology.orchestra.run.smithi069:> openssl req -nodes -newkey rsa:2048 -keyout /home/ubuntu/cephtest/ca/rgw.client.1.key -out /home/ubuntu/cephtest/ca/rgw.client.1.csr -subj /CN=smithi174.front.sepia.ceph.com 2023-09-22T13:36:43.871 INFO:teuthology.orchestra.run.smithi069.stderr:Generating a RSA private key 2023-09-22T13:36:43.928 INFO:teuthology.orchestra.run.smithi069.stderr:..........................................................+++++ 2023-09-22T13:36:43.945 INFO:teuthology.orchestra.run.smithi069.stderr:.................+++++ 2023-09-22T13:36:43.945 INFO:teuthology.orchestra.run.smithi069.stderr:writing new private key to '/home/ubuntu/cephtest/ca/rgw.client.1.key' 2023-09-22T13:36:43.945 INFO:teuthology.orchestra.run.smithi069.stderr:----- 2023-09-22T13:36:43.947 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:36:43.947 DEBUG:teuthology.orchestra.run.smithi069:> dd of=/home/ubuntu/cephtest/ca/rgw.client.1.ext 2023-09-22T13:36:44.005 DEBUG:teuthology.orchestra.run.smithi069:> openssl x509 -req -in /home/ubuntu/cephtest/ca/rgw.client.1.csr -CA /home/ubuntu/cephtest/ca/root.crt -CAkey /home/ubuntu/cephtest/ca/root.key -CAcreateserial -out /home/ubuntu/cephtest/ca/rgw.client.1.crt -days 365 -sha256 -extfile /home/ubuntu/cephtest/ca/rgw.client.1.ext 2023-09-22T13:36:44.069 INFO:teuthology.orchestra.run.smithi069.stderr:Signature ok 2023-09-22T13:36:44.069 INFO:teuthology.orchestra.run.smithi069.stderr:subject=CN = smithi174.front.sepia.ceph.com 2023-09-22T13:36:44.069 INFO:teuthology.orchestra.run.smithi069.stderr:Getting CA Private Key 2023-09-22T13:36:44.075 DEBUG:teuthology.orchestra.run.smithi069:> rm /home/ubuntu/cephtest/ca/rgw.client.1.csr /home/ubuntu/cephtest/ca/root.srl /home/ubuntu/cephtest/ca/rgw.client.1.ext 2023-09-22T13:36:44.133 DEBUG:teuthology.orchestra.run.smithi069:> openssl verify -CAfile /home/ubuntu/cephtest/ca/root.crt /home/ubuntu/cephtest/ca/rgw.client.1.crt 2023-09-22T13:36:44.193 INFO:teuthology.orchestra.run.smithi069.stdout:/home/ubuntu/cephtest/ca/rgw.client.1.crt: OK 2023-09-22T13:36:44.193 INFO:tasks.openssl_keys:copying from ubuntu@smithi069.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/rgw.client.1.crt to ubuntu@smithi174.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/rgw.client.1.crt... 2023-09-22T13:36:44.254 DEBUG:teuthology.orchestra.remote:smithi069:/home/ubuntu/cephtest/ca/rgw.client.1.crt is 1KB 2023-09-22T13:36:44.299 INFO:tasks.openssl_keys:copying from ubuntu@smithi069.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/rgw.client.1.key to ubuntu@smithi174.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/rgw.client.1.key... 2023-09-22T13:36:44.319 DEBUG:teuthology.orchestra.remote:smithi069:/home/ubuntu/cephtest/ca/rgw.client.1.key is 2KB 2023-09-22T13:36:44.389 DEBUG:teuthology.orchestra.run.smithi069:> rm /home/ubuntu/cephtest/ca/rgw.client.1.crt /home/ubuntu/cephtest/ca/rgw.client.1.key 2023-09-22T13:36:44.406 DEBUG:teuthology.orchestra.run.smithi174:> openssl verify /home/ubuntu/cephtest/ca/rgw.client.1.crt 2023-09-22T13:36:44.446 INFO:teuthology.orchestra.run.smithi174.stdout:/home/ubuntu/cephtest/ca/rgw.client.1.crt: OK 2023-09-22T13:36:44.446 DEBUG:teuthology.orchestra.run.smithi174:> cat /home/ubuntu/cephtest/ca/rgw.client.1.key >> /home/ubuntu/cephtest/ca/rgw.client.1.crt 2023-09-22T13:36:44.504 INFO:teuthology.run_tasks:Running task rgw... 2023-09-22T13:36:44.516 DEBUG:tasks.rgw:config is {'client.0': {'ssl certificate': 'rgw.client.0'}, 'client.1': {'ssl certificate': 'rgw.client.1'}} 2023-09-22T13:36:44.516 DEBUG:tasks.rgw:client list is dict_keys(['client.0', 'client.1']) 2023-09-22T13:36:44.516 INFO:tasks.rgw:Creating data pools 2023-09-22T13:36:44.517 DEBUG:tasks.rgw:Obtaining remote for client client.0 2023-09-22T13:36:44.517 DEBUG:teuthology.orchestra.run.smithi069:> sudo ceph osd erasure-code-profile set client.0 crush-failure-domain=osd k=3 m=1 name=testprofile --cluster ceph 2023-09-22T13:36:45.555 DEBUG:teuthology.orchestra.run.smithi069:> sudo ceph osd pool create default.rgw.buckets.data 64 64 erasure client.0 --cluster ceph 2023-09-22T13:36:47.564 INFO:teuthology.orchestra.run.smithi069.stderr:pool 'default.rgw.buckets.data' created 2023-09-22T13:36:47.576 DEBUG:teuthology.orchestra.run.smithi069:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster ceph 2023-09-22T13:36:48.588 INFO:teuthology.orchestra.run.smithi069.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2023-09-22T13:36:48.603 DEBUG:teuthology.orchestra.run.smithi069:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster ceph 2023-09-22T13:36:49.565 INFO:teuthology.orchestra.run.smithi069.stderr:pool 'default.rgw.buckets.index' created 2023-09-22T13:36:49.582 DEBUG:teuthology.orchestra.run.smithi069:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster ceph 2023-09-22T13:36:50.588 INFO:teuthology.orchestra.run.smithi069.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2023-09-22T13:36:50.599 DEBUG:tasks.rgw:Obtaining remote for client client.1 2023-09-22T13:36:50.600 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph osd erasure-code-profile set client.1 crush-failure-domain=osd k=3 m=1 name=testprofile --cluster ceph 2023-09-22T13:36:51.596 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph osd pool create default.rgw.buckets.data 64 64 erasure client.1 --cluster ceph 2023-09-22T13:36:51.892 INFO:teuthology.orchestra.run.smithi174.stderr:pool 'default.rgw.buckets.data' already exists 2023-09-22T13:36:51.902 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster ceph 2023-09-22T13:36:52.582 INFO:teuthology.orchestra.run.smithi174.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2023-09-22T13:36:52.594 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster ceph 2023-09-22T13:36:52.883 INFO:teuthology.orchestra.run.smithi174.stderr:pool 'default.rgw.buckets.index' already exists 2023-09-22T13:36:52.894 DEBUG:teuthology.orchestra.run.smithi174:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster ceph 2023-09-22T13:36:53.597 INFO:teuthology.orchestra.run.smithi174.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2023-09-22T13:36:53.608 DEBUG:tasks.rgw:Pools created 2023-09-22T13:36:53.608 INFO:tasks.rgw:Configuring compression type = random 2023-09-22T13:36:53.608 INFO:tasks.util.rgw:rgwadmin: client.0 : ['user', 'list'] 2023-09-22T13:36:53.609 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'user', 'list'] 2023-09-22T13:36:53.609 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph user list 2023-09-22T13:36:53.712 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:36:53.712 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:36:53.720 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:53.718+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.607 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.605+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.607 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.606+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.610 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.608+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.610 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.608+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.610 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.609+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.610 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.609+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.611 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.610+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.625 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.623+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.626 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.624+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.626 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.624+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.626 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.624+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.626 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.624+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.626 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.624+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.626 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.624+0000 7fe0e18be500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:36:55.626 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.624+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.626 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.624+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.626 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.624+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.627 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.625+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.627 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.625+0000 7fe0e18be500 10 Creating default zonegroup 2023-09-22T13:36:55.627 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.625+0000 7fe0e18be500 20 get_system_obj_state: rctx=0x7ffe72150460 obj=:.pools.avail state=0x55dbdbcf4810 s->prefetch_data=0 2023-09-22T13:36:55.627 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.625+0000 7fe0e18be500 10 couldn't find old data placement pools config, setting up new ones for the zone 2023-09-22T13:36:55.632 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.630+0000 7fe0cd5a1700 0 -- 172.21.15.69:0/3558152380 >> [v2:172.21.15.174:6824/145172,v1:172.21.15.174:6825/145172] conn(0x7fe0b400f8f0 msgr2=0x7fe0b4078040 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:36:55.635 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.634+0000 7fe0e18be500 10 WARNING: store->list_zones() returned r=-2 2023-09-22T13:36:55.636 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.634+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.636 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.634+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.640 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.638+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.640 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.639+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.640 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.639+0000 7fe0e18be500 10 could not read realm id: (2) No such file or directory 2023-09-22T13:36:55.641 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.639+0000 7fe0e18be500 10 WARNING: failed to set zone as default, r=-22 2023-09-22T13:36:55.641 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.639+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.641 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.639+0000 7fe0cc59f700 0 -- 172.21.15.69:0/3558152380 >> [v2:172.21.15.174:6808/145141,v1:172.21.15.174:6809/145141] conn(0x55dbdbe3a4b0 msgr2=0x55dbdbe3c8a0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:36:55.641 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.639+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.643 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.642+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.645 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.643+0000 7fe0e18be500 20 rados_obj.operate() r=0 bl.length=873 2023-09-22T13:36:55.645 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.643+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.644+0000 7fe0e18be500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:36:55.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.644+0000 7fe0e18be500 20 zonegroup default 2023-09-22T13:36:55.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.644+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.645+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:55.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.645+0000 7fe0e18be500 10 Cannot find current period zone using local zone 2023-09-22T13:36:55.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.645+0000 7fe0e18be500 10 Using default name default 2023-09-22T13:36:55.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.645+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.645+0000 7fe0e18be500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:36:55.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.645+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:55.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.646+0000 7fe0e18be500 20 rados_obj.operate() r=0 bl.length=873 2023-09-22T13:36:55.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.646+0000 7fe0e18be500 20 zone default found 2023-09-22T13:36:55.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:55.646+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:57.611 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:57.609+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:57.611 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:57.609+0000 7fe0e18be500 20 rados->read ofs=0 len=0 2023-09-22T13:36:57.611 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:57.610+0000 7fe0e18be500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:36:57.611 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:57.610+0000 7fe0e18be500 20 started sync module instance, tier type = 2023-09-22T13:36:57.611 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:57.610+0000 7fe0e18be500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:36:59.635 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.633+0000 7fe0e18be500 20 add_watcher() i=0 2023-09-22T13:36:59.635 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.633+0000 7fe0e18be500 20 add_watcher() i=1 2023-09-22T13:36:59.635 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.633+0000 7fe0e18be500 20 add_watcher() i=2 2023-09-22T13:36:59.635 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.633+0000 7fe0e18be500 20 add_watcher() i=3 2023-09-22T13:36:59.635 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.633+0000 7fe0e18be500 20 add_watcher() i=4 2023-09-22T13:36:59.635 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.633+0000 7fe0e18be500 20 add_watcher() i=5 2023-09-22T13:36:59.636 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.633+0000 7fe0e18be500 20 add_watcher() i=6 2023-09-22T13:36:59.638 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.636+0000 7fe0e18be500 20 add_watcher() i=7 2023-09-22T13:36:59.638 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.636+0000 7fe0e18be500 2 all 8 watchers are set, enabling cache 2023-09-22T13:36:59.638 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.636+0000 7fe0e18be500 5 tl::expected, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):409: oid=data_loggenerations_metadata not found 2023-09-22T13:36:59.638 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.636+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.0 2023-09-22T13:36:59.638 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.636+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.639 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.637+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.0 does not exist 2023-09-22T13:36:59.639 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.637+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.1 2023-09-22T13:36:59.639 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.637+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.639 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.637+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.1 does not exist 2023-09-22T13:36:59.639 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.637+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.2 2023-09-22T13:36:59.639 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.637+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.639 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.637+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.2 does not exist 2023-09-22T13:36:59.639 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.637+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.3 2023-09-22T13:36:59.640 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.637+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.641 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.3 does not exist 2023-09-22T13:36:59.641 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.4 2023-09-22T13:36:59.641 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.641 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.4 does not exist 2023-09-22T13:36:59.641 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.5 2023-09-22T13:36:59.641 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.641 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.5 does not exist 2023-09-22T13:36:59.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.6 2023-09-22T13:36:59.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.6 does not exist 2023-09-22T13:36:59.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.7 2023-09-22T13:36:59.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.638+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.7 does not exist 2023-09-22T13:36:59.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.8 2023-09-22T13:36:59.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.8 does not exist 2023-09-22T13:36:59.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.9 2023-09-22T13:36:59.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.643 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.9 does not exist 2023-09-22T13:36:59.643 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.10 2023-09-22T13:36:59.643 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.643 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.10 does not exist 2023-09-22T13:36:59.643 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.11 2023-09-22T13:36:59.643 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.639+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.643 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.640+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.11 does not exist 2023-09-22T13:36:59.643 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.640+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.12 2023-09-22T13:36:59.643 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.640+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.645 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.640+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.12 does not exist 2023-09-22T13:36:59.645 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.640+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.13 2023-09-22T13:36:59.645 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.640+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.645 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.640+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.13 does not exist 2023-09-22T13:36:59.645 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.640+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.14 2023-09-22T13:36:59.645 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.640+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.14 does not exist 2023-09-22T13:36:59.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.15 2023-09-22T13:36:59.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.15 does not exist 2023-09-22T13:36:59.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.16 2023-09-22T13:36:59.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.16 does not exist 2023-09-22T13:36:59.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.17 2023-09-22T13:36:59.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.17 does not exist 2023-09-22T13:36:59.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.18 2023-09-22T13:36:59.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.641+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.18 does not exist 2023-09-22T13:36:59.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.19 2023-09-22T13:36:59.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.19 does not exist 2023-09-22T13:36:59.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.20 2023-09-22T13:36:59.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.20 does not exist 2023-09-22T13:36:59.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.21 2023-09-22T13:36:59.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.21 does not exist 2023-09-22T13:36:59.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.22 2023-09-22T13:36:59.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.642+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.643+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.22 does not exist 2023-09-22T13:36:59.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.643+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.23 2023-09-22T13:36:59.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.643+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.643+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.23 does not exist 2023-09-22T13:36:59.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.643+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.24 2023-09-22T13:36:59.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.643+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.643+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.24 does not exist 2023-09-22T13:36:59.648 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.643+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.25 2023-09-22T13:36:59.649 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.643+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.649 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.25 does not exist 2023-09-22T13:36:59.649 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.26 2023-09-22T13:36:59.649 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.651 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.26 does not exist 2023-09-22T13:36:59.652 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.27 2023-09-22T13:36:59.652 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.652 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.27 does not exist 2023-09-22T13:36:59.652 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.28 2023-09-22T13:36:59.652 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.652 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.28 does not exist 2023-09-22T13:36:59.652 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.29 2023-09-22T13:36:59.652 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.644+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.652 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.645+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.29 does not exist 2023-09-22T13:36:59.652 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.645+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.30 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.645+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.645+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.30 does not exist 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.645+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.31 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.645+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.645+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.31 does not exist 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.645+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.32 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.645+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.32 does not exist 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.33 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.33 does not exist 2023-09-22T13:36:59.653 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.34 2023-09-22T13:36:59.654 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.654 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.34 does not exist 2023-09-22T13:36:59.654 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.35 2023-09-22T13:36:59.654 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.654 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.35 does not exist 2023-09-22T13:36:59.654 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.36 2023-09-22T13:36:59.654 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.646+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.654 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.36 does not exist 2023-09-22T13:36:59.654 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.37 2023-09-22T13:36:59.654 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.37 does not exist 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.38 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.38 does not exist 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.39 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.39 does not exist 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.40 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.647+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.648+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.40 does not exist 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.648+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.41 2023-09-22T13:36:59.655 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.648+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.656 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.648+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.41 does not exist 2023-09-22T13:36:59.656 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.648+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.42 2023-09-22T13:36:59.656 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.648+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.656 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.648+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.42 does not exist 2023-09-22T13:36:59.656 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.648+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.43 2023-09-22T13:36:59.656 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.648+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.656 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.43 does not exist 2023-09-22T13:36:59.656 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.44 2023-09-22T13:36:59.656 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.656 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.44 does not exist 2023-09-22T13:36:59.656 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.45 2023-09-22T13:36:59.657 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.657 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.45 does not exist 2023-09-22T13:36:59.657 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.46 2023-09-22T13:36:59.657 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.657 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.46 does not exist 2023-09-22T13:36:59.657 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.47 2023-09-22T13:36:59.657 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.649+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.657 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.47 does not exist 2023-09-22T13:36:59.657 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.48 2023-09-22T13:36:59.657 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.658 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.48 does not exist 2023-09-22T13:36:59.658 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.49 2023-09-22T13:36:59.658 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.658 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.49 does not exist 2023-09-22T13:36:59.658 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.50 2023-09-22T13:36:59.658 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.663 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.50 does not exist 2023-09-22T13:36:59.663 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.51 2023-09-22T13:36:59.663 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.650+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.663 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.51 does not exist 2023-09-22T13:36:59.663 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.52 2023-09-22T13:36:59.663 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.663 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.52 does not exist 2023-09-22T13:36:59.663 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.53 2023-09-22T13:36:59.664 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.664 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.53 does not exist 2023-09-22T13:36:59.664 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.54 2023-09-22T13:36:59.664 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.664 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.54 does not exist 2023-09-22T13:36:59.664 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.55 2023-09-22T13:36:59.664 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.651+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.664 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.652+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.55 does not exist 2023-09-22T13:36:59.664 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.652+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.56 2023-09-22T13:36:59.664 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.652+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.664 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.652+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.56 does not exist 2023-09-22T13:36:59.665 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.652+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.57 2023-09-22T13:36:59.665 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.652+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.665 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.652+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.57 does not exist 2023-09-22T13:36:59.665 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.652+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.58 2023-09-22T13:36:59.665 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.652+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.665 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.58 does not exist 2023-09-22T13:36:59.665 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.59 2023-09-22T13:36:59.665 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.665 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.59 does not exist 2023-09-22T13:36:59.665 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.60 2023-09-22T13:36:59.665 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.666 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.60 does not exist 2023-09-22T13:36:59.666 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.61 2023-09-22T13:36:59.666 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.666 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.61 does not exist 2023-09-22T13:36:59.666 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.62 2023-09-22T13:36:59.666 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.653+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.666 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.62 does not exist 2023-09-22T13:36:59.666 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.63 2023-09-22T13:36:59.666 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.666 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.63 does not exist 2023-09-22T13:36:59.666 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.64 2023-09-22T13:36:59.667 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.667 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.64 does not exist 2023-09-22T13:36:59.667 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.65 2023-09-22T13:36:59.667 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.667 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.65 does not exist 2023-09-22T13:36:59.667 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.66 2023-09-22T13:36:59.667 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.667 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.66 does not exist 2023-09-22T13:36:59.667 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.67 2023-09-22T13:36:59.667 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.654+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.667 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.67 does not exist 2023-09-22T13:36:59.668 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.68 2023-09-22T13:36:59.668 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.668 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.68 does not exist 2023-09-22T13:36:59.668 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.69 2023-09-22T13:36:59.668 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.668 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.69 does not exist 2023-09-22T13:36:59.668 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.70 2023-09-22T13:36:59.668 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.668 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.70 does not exist 2023-09-22T13:36:59.668 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.71 2023-09-22T13:36:59.668 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.669 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.71 does not exist 2023-09-22T13:36:59.669 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.72 2023-09-22T13:36:59.669 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.655+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.669 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.656+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.72 does not exist 2023-09-22T13:36:59.669 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.656+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.73 2023-09-22T13:36:59.669 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.656+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.669 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.656+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.73 does not exist 2023-09-22T13:36:59.669 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.656+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.74 2023-09-22T13:36:59.669 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.656+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.669 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.656+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.74 does not exist 2023-09-22T13:36:59.669 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.656+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.75 2023-09-22T13:36:59.670 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.656+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.670 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.75 does not exist 2023-09-22T13:36:59.670 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.76 2023-09-22T13:36:59.670 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.670 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.76 does not exist 2023-09-22T13:36:59.670 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.77 2023-09-22T13:36:59.670 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.670 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.77 does not exist 2023-09-22T13:36:59.670 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.78 2023-09-22T13:36:59.670 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.670 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.78 does not exist 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.79 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.657+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.658+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.79 does not exist 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.658+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.80 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.658+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.658+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.80 does not exist 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.658+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.81 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.658+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.658+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.81 does not exist 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.658+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.82 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.658+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.671 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.659+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.82 does not exist 2023-09-22T13:36:59.672 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.659+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.83 2023-09-22T13:36:59.672 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.659+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.672 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.659+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.83 does not exist 2023-09-22T13:36:59.672 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.659+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.84 2023-09-22T13:36:59.672 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.659+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.672 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.659+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.84 does not exist 2023-09-22T13:36:59.672 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.659+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.85 2023-09-22T13:36:59.672 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.659+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.672 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.85 does not exist 2023-09-22T13:36:59.672 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.86 2023-09-22T13:36:59.672 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.673 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.86 does not exist 2023-09-22T13:36:59.673 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.87 2023-09-22T13:36:59.673 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.673 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.87 does not exist 2023-09-22T13:36:59.673 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.88 2023-09-22T13:36:59.673 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.673 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.88 does not exist 2023-09-22T13:36:59.673 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.89 2023-09-22T13:36:59.673 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.673 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.89 does not exist 2023-09-22T13:36:59.673 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.90 2023-09-22T13:36:59.674 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.660+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.674 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.661+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.90 does not exist 2023-09-22T13:36:59.674 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.661+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.91 2023-09-22T13:36:59.674 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.661+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.674 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.661+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.91 does not exist 2023-09-22T13:36:59.674 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.661+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.92 2023-09-22T13:36:59.674 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.661+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.674 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.661+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.92 does not exist 2023-09-22T13:36:59.674 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.661+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.93 2023-09-22T13:36:59.674 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.661+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.678 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.93 does not exist 2023-09-22T13:36:59.678 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.94 2023-09-22T13:36:59.678 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.678 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.94 does not exist 2023-09-22T13:36:59.678 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.95 2023-09-22T13:36:59.678 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.678 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.95 does not exist 2023-09-22T13:36:59.679 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.96 2023-09-22T13:36:59.679 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.679 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.96 does not exist 2023-09-22T13:36:59.679 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.97 2023-09-22T13:36:59.679 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.662+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.679 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.97 does not exist 2023-09-22T13:36:59.679 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.98 2023-09-22T13:36:59.679 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.679 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.98 does not exist 2023-09-22T13:36:59.679 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.99 2023-09-22T13:36:59.679 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.99 does not exist 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.100 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.100 does not exist 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.101 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.101 does not exist 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.102 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.663+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.102 does not exist 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.103 2023-09-22T13:36:59.680 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.681 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.103 does not exist 2023-09-22T13:36:59.681 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.104 2023-09-22T13:36:59.681 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.681 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.104 does not exist 2023-09-22T13:36:59.681 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.105 2023-09-22T13:36:59.681 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.681 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.105 does not exist 2023-09-22T13:36:59.681 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.106 2023-09-22T13:36:59.681 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.681 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.106 does not exist 2023-09-22T13:36:59.681 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.107 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.664+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.107 does not exist 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.108 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.108 does not exist 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.109 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.109 does not exist 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.110 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.110 does not exist 2023-09-22T13:36:59.682 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.111 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.665+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.111 does not exist 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.112 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.112 does not exist 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.113 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.113 does not exist 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.114 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.114 does not exist 2023-09-22T13:36:59.683 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.115 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.666+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.115 does not exist 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.116 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.116 does not exist 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.117 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.117 does not exist 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.118 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.118 does not exist 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.119 2023-09-22T13:36:59.684 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.119 does not exist 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.120 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.667+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.120 does not exist 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.121 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.121 does not exist 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.122 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.122 does not exist 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.123 2023-09-22T13:36:59.685 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.686 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.123 does not exist 2023-09-22T13:36:59.686 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.124 2023-09-22T13:36:59.686 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.668+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.686 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.669+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.124 does not exist 2023-09-22T13:36:59.686 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.669+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.125 2023-09-22T13:36:59.686 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.669+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.686 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.669+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.125 does not exist 2023-09-22T13:36:59.686 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.669+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.126 2023-09-22T13:36:59.686 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.669+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.686 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.669+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.126 does not exist 2023-09-22T13:36:59.686 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.669+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.127 2023-09-22T13:36:59.687 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.669+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.687 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.669+0000 7fe0e18be500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.127 does not exist 2023-09-22T13:36:59.687 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.669+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::create(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, std::optional >, bool, uint64_t, uint64_t):1206 entering 2023-09-22T13:36:59.687 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.671+0000 7fe0e18be500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:36:59.687 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.674+0000 7fe09e7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:36:59.687 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.674+0000 7fe0e18be500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:36:59.687 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:36:59.674+0000 7fe0e18be500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:02.607 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.605+0000 7fe0e18be500 10 rgw_init_ioctx warning: failed to set recovery_priority on default.rgw.meta 2023-09-22T13:37:02.607 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.605+0000 7fe0e18be500 5 note: GC not initialized 2023-09-22T13:37:02.609 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.607+0000 7fe00d7fa700 20 reqs_thread_entry: start 2023-09-22T13:37:02.609 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.607+0000 7fdffaffd700 20 reqs_thread_entry: start 2023-09-22T13:37:02.609 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.607+0000 7fe0e18be500 20 init_complete bucket index max shards: 11 2023-09-22T13:37:02.609 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.607+0000 7fe0e18be500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:37:02.609 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.607+0000 7fdff97fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:37:32 2023 2023-09-22T13:37:02.609 INFO:teuthology.orchestra.run.smithi069.stderr: 2023-09-22T13:37:02.609 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.608+0000 7fdff8ff9700 20 reqs_thread_entry: start 2023-09-22T13:37:02.638 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.636+0000 7fe0e18be500 20 remove_watcher() i=0 2023-09-22T13:37:02.638 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.636+0000 7fe0e18be500 2 removed watcher, disabling cache 2023-09-22T13:37:02.639 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.637+0000 7fe0e18be500 20 remove_watcher() i=1 2023-09-22T13:37:02.640 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.639+0000 7fe0e18be500 20 remove_watcher() i=2 2023-09-22T13:37:02.642 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.640+0000 7fe0e18be500 20 remove_watcher() i=3 2023-09-22T13:37:02.643 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.641+0000 7fe0e18be500 20 remove_watcher() i=4 2023-09-22T13:37:02.644 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.643+0000 7fe0e18be500 20 remove_watcher() i=5 2023-09-22T13:37:02.645 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.644+0000 7fe0e18be500 20 remove_watcher() i=6 2023-09-22T13:37:02.646 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.645+0000 7fe0e18be500 20 remove_watcher() i=7 2023-09-22T13:37:02.647 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.646+0000 7fe09e7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:37:02.651 INFO:teuthology.orchestra.run.smithi069.stdout:[] 2023-09-22T13:37:02.653 DEBUG:tasks.util.rgw: json result: [] 2023-09-22T13:37:02.653 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2023-09-22T13:37:02.653 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2023-09-22T13:37:02.654 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zone placement modify --rgw-zone default --placement-id default-placement --compression random 2023-09-22T13:37:02.701 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:02.701 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:02.707 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.706+0000 7fefd96b8500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:02.708 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.706+0000 7fefd96b8500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:02.712 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.711+0000 7fef0dffb700 20 reqs_thread_entry: start 2023-09-22T13:37:02.713 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.711+0000 7fefd96b8500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.714 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.712+0000 7fefd96b8500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:02.714 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.712+0000 7fefd96b8500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.715 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.713+0000 7fefd96b8500 20 rados_obj.operate() r=0 bl.length=873 2023-09-22T13:37:02.715 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.713+0000 7fefd96b8500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.715 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.714+0000 7fefd96b8500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:02.716 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.714+0000 7fefd96b8500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.716 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.715+0000 7fefd96b8500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:02.716 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.715+0000 7fefd96b8500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.716 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:02.715+0000 7fefd96b8500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:37:02.721 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2023-09-22T13:37:02.721 INFO:teuthology.orchestra.run.smithi069.stdout: "id": "f54d190c-30d0-4b87-88c4-5bcc7de96de4", 2023-09-22T13:37:02.721 INFO:teuthology.orchestra.run.smithi069.stdout: "name": "default", 2023-09-22T13:37:02.721 INFO:teuthology.orchestra.run.smithi069.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:37:02.722 INFO:teuthology.orchestra.run.smithi069.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:37:02.722 INFO:teuthology.orchestra.run.smithi069.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:37:02.722 INFO:teuthology.orchestra.run.smithi069.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:37:02.722 INFO:teuthology.orchestra.run.smithi069.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:37:02.722 INFO:teuthology.orchestra.run.smithi069.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:37:02.722 INFO:teuthology.orchestra.run.smithi069.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:37:02.722 INFO:teuthology.orchestra.run.smithi069.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:37:02.722 INFO:teuthology.orchestra.run.smithi069.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:37:02.722 INFO:teuthology.orchestra.run.smithi069.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:37:02.722 INFO:teuthology.orchestra.run.smithi069.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:37:02.722 INFO:teuthology.orchestra.run.smithi069.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: "system_key": { 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: "access_key": "", 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: "secret_key": "" 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: "placement_pools": [ 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: { 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: "key": "default-placement", 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: "val": { 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:37:02.723 INFO:teuthology.orchestra.run.smithi069.stdout: "storage_classes": { 2023-09-22T13:37:02.724 INFO:teuthology.orchestra.run.smithi069.stdout: "STANDARD": { 2023-09-22T13:37:02.724 INFO:teuthology.orchestra.run.smithi069.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:37:02.724 INFO:teuthology.orchestra.run.smithi069.stdout: "compression_type": "random" 2023-09-22T13:37:02.724 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:02.724 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:37:02.724 INFO:teuthology.orchestra.run.smithi069.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:37:02.724 INFO:teuthology.orchestra.run.smithi069.stdout: "index_type": 0 2023-09-22T13:37:02.724 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:02.724 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:02.724 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2023-09-22T13:37:02.725 INFO:teuthology.orchestra.run.smithi069.stdout: "realm_id": "", 2023-09-22T13:37:02.725 INFO:teuthology.orchestra.run.smithi069.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:37:02.725 INFO:teuthology.orchestra.run.smithi069.stdout:} 2023-09-22T13:37:02.726 DEBUG:tasks.util.rgw: json result: {'id': 'f54d190c-30d0-4b87-88c4-5bcc7de96de4', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:37:02.726 INFO:tasks.util.rgw:rgwadmin: client.1 : ['user', 'list'] 2023-09-22T13:37:02.726 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'user', 'list'] 2023-09-22T13:37:02.726 DEBUG:teuthology.orchestra.run.smithi174:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph user list 2023-09-22T13:37:02.841 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:02.841 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:02.847 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.846+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.849 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.848+0000 7fd28dbd0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:02.849 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.848+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.850 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.849+0000 7fd28dbd0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:02.850 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.849+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.850 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.849+0000 7fd28dbd0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:02.850 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.849+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.851 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.850+0000 7fd28dbd0500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:02.863 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.862+0000 7fd28dbd0500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:37:02.863 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.862+0000 7fd28dbd0500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:37:02.863 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.862+0000 7fd28dbd0500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:37:02.863 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.862+0000 7fd28dbd0500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:37:02.864 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.862+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.864 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.862+0000 7fd28dbd0500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:02.864 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.862+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.867 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.865+0000 7fd28dbd0500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:37:02.867 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.866+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.867 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.866+0000 7fd28dbd0500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:02.867 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.866+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.868 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.866+0000 7fd28dbd0500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:37:02.868 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.867+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.868 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.867+0000 7fd28dbd0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:02.868 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.867+0000 7fd28dbd0500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:37:02.868 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.867+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.869 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.867+0000 7fd28dbd0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:02.869 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.867+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.869 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.868+0000 7fd28dbd0500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:02.869 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.868+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.870 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.868+0000 7fd28dbd0500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:37:02.870 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.868+0000 7fd28dbd0500 20 zonegroup default 2023-09-22T13:37:02.870 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.868+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.870 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.868+0000 7fd28dbd0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:02.871 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.868+0000 7fd28dbd0500 10 Cannot find current period zone using local zone 2023-09-22T13:37:02.871 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.868+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.871 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.869+0000 7fd28dbd0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:02.871 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.869+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.871 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.869+0000 7fd28dbd0500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:02.871 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.869+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.871 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.869+0000 7fd28dbd0500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:37:02.871 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.869+0000 7fd28dbd0500 20 zone default found 2023-09-22T13:37:02.871 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.869+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.871 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.870+0000 7fd28dbd0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:02.871 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.870+0000 7fd28dbd0500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.872 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.870+0000 7fd28dbd0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:02.872 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.870+0000 7fd28dbd0500 20 started sync module instance, tier type = 2023-09-22T13:37:02.872 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.870+0000 7fd28dbd0500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:37:02.892 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.891+0000 7fd28dbd0500 20 add_watcher() i=0 2023-09-22T13:37:02.892 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.891+0000 7fd28dbd0500 20 add_watcher() i=1 2023-09-22T13:37:02.892 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.891+0000 7fd28dbd0500 20 add_watcher() i=2 2023-09-22T13:37:02.892 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.891+0000 7fd28dbd0500 20 add_watcher() i=3 2023-09-22T13:37:02.893 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.891+0000 7fd28dbd0500 20 add_watcher() i=4 2023-09-22T13:37:02.893 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.891+0000 7fd28dbd0500 20 add_watcher() i=5 2023-09-22T13:37:02.893 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.891+0000 7fd28dbd0500 20 add_watcher() i=6 2023-09-22T13:37:02.894 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.894+0000 7fd28dbd0500 20 add_watcher() i=7 2023-09-22T13:37:02.894 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.894+0000 7fd28dbd0500 2 all 8 watchers are set, enabling cache 2023-09-22T13:37:02.897 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.896+0000 7fd24a7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:37:02.897 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.896+0000 7fd28dbd0500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:02.897 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.896+0000 7fd28dbd0500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:02.898 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.897+0000 7fd2790b2700 0 -- >> [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] conn(0x557c52148470 msgr2=0x557c521489b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2023-09-22T13:37:02.902 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.901+0000 7fd28dbd0500 5 note: GC not initialized 2023-09-22T13:37:02.903 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.902+0000 7fd1b97fa700 20 reqs_thread_entry: start 2023-09-22T13:37:02.903 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.902+0000 7fd1b2ffd700 20 reqs_thread_entry: start 2023-09-22T13:37:02.903 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.902+0000 7fd28dbd0500 20 init_complete bucket index max shards: 11 2023-09-22T13:37:02.903 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.902+0000 7fd28dbd0500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:37:02.903 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.902+0000 7fd1b17fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:37:33 2023 2023-09-22T13:37:02.904 INFO:teuthology.orchestra.run.smithi174.stderr: 2023-09-22T13:37:02.904 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.902+0000 7fd1b0ff9700 20 reqs_thread_entry: start 2023-09-22T13:37:02.904 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.903+0000 7fd2790b2700 0 -- 172.21.15.174:0/2867960927 >> [v2:172.21.15.69:6808/145166,v1:172.21.15.69:6809/145166] conn(0x557c52128e10 msgr2=0x557c5212b250 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:37:02.914 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.913+0000 7fd28dbd0500 20 remove_watcher() i=0 2023-09-22T13:37:02.914 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.913+0000 7fd28dbd0500 2 removed watcher, disabling cache 2023-09-22T13:37:02.915 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.914+0000 7fd28dbd0500 20 remove_watcher() i=1 2023-09-22T13:37:02.916 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.915+0000 7fd28dbd0500 20 remove_watcher() i=2 2023-09-22T13:37:02.917 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.916+0000 7fd28dbd0500 20 remove_watcher() i=3 2023-09-22T13:37:02.918 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.917+0000 7fd28dbd0500 20 remove_watcher() i=4 2023-09-22T13:37:02.919 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.918+0000 7fd28dbd0500 20 remove_watcher() i=5 2023-09-22T13:37:02.920 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.919+0000 7fd28dbd0500 20 remove_watcher() i=6 2023-09-22T13:37:02.921 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.920+0000 7fd28dbd0500 20 remove_watcher() i=7 2023-09-22T13:37:02.922 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.921+0000 7fd24a7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:37:02.925 INFO:teuthology.orchestra.run.smithi174.stdout:[] 2023-09-22T13:37:02.927 DEBUG:tasks.util.rgw: json result: [] 2023-09-22T13:37:02.927 INFO:tasks.util.rgw:rgwadmin: client.1 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2023-09-22T13:37:02.927 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2023-09-22T13:37:02.927 DEBUG:teuthology.orchestra.run.smithi174:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph zone placement modify --rgw-zone default --placement-id default-placement --compression random 2023-09-22T13:37:02.973 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:02.973 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:02.979 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.978+0000 7fd1aff02500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:02.979 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.978+0000 7fd1aff02500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:02.983 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.982+0000 7fd0dffff700 20 reqs_thread_entry: start 2023-09-22T13:37:02.983 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.982+0000 7fd1aff02500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.984 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.983+0000 7fd1aff02500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:02.984 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.983+0000 7fd1aff02500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.984 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.984+0000 7fd1aff02500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:37:02.985 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.984+0000 7fd1aff02500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.985 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.984+0000 7fd1aff02500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:02.985 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.984+0000 7fd1aff02500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.985 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.984+0000 7fd1aff02500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:02.985 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.984+0000 7fd1aff02500 20 rados->read ofs=0 len=0 2023-09-22T13:37:02.986 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:02.985+0000 7fd1aff02500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:37:02.990 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2023-09-22T13:37:02.990 INFO:teuthology.orchestra.run.smithi174.stdout: "id": "f54d190c-30d0-4b87-88c4-5bcc7de96de4", 2023-09-22T13:37:02.990 INFO:teuthology.orchestra.run.smithi174.stdout: "name": "default", 2023-09-22T13:37:02.990 INFO:teuthology.orchestra.run.smithi174.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:37:02.990 INFO:teuthology.orchestra.run.smithi174.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:37:02.990 INFO:teuthology.orchestra.run.smithi174.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:37:02.991 INFO:teuthology.orchestra.run.smithi174.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:37:02.991 INFO:teuthology.orchestra.run.smithi174.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:37:02.991 INFO:teuthology.orchestra.run.smithi174.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:37:02.991 INFO:teuthology.orchestra.run.smithi174.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:37:02.991 INFO:teuthology.orchestra.run.smithi174.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:37:02.991 INFO:teuthology.orchestra.run.smithi174.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:37:02.991 INFO:teuthology.orchestra.run.smithi174.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:37:02.991 INFO:teuthology.orchestra.run.smithi174.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:37:02.991 INFO:teuthology.orchestra.run.smithi174.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:37:02.991 INFO:teuthology.orchestra.run.smithi174.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "system_key": { 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "access_key": "", 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "secret_key": "" 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "placement_pools": [ 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: { 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "key": "default-placement", 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "val": { 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "storage_classes": { 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "STANDARD": { 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:37:02.992 INFO:teuthology.orchestra.run.smithi174.stdout: "compression_type": "random" 2023-09-22T13:37:02.993 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:02.993 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2023-09-22T13:37:02.993 INFO:teuthology.orchestra.run.smithi174.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:37:02.993 INFO:teuthology.orchestra.run.smithi174.stdout: "index_type": 0 2023-09-22T13:37:02.993 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:02.993 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:02.993 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2023-09-22T13:37:02.993 INFO:teuthology.orchestra.run.smithi174.stdout: "realm_id": "", 2023-09-22T13:37:02.993 INFO:teuthology.orchestra.run.smithi174.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:37:02.993 INFO:teuthology.orchestra.run.smithi174.stdout:} 2023-09-22T13:37:02.994 DEBUG:tasks.util.rgw: json result: {'id': 'f54d190c-30d0-4b87-88c4-5bcc7de96de4', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:37:02.994 INFO:tasks.util.rgw:rgwadmin: client.0 : ['user', 'list'] 2023-09-22T13:37:02.994 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'user', 'list'] 2023-09-22T13:37:02.995 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph user list 2023-09-22T13:37:03.034 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:03.034 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:03.041 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.039+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.042 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.040+0000 7f972780a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.042 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.040+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.042 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.041+0000 7f972780a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.043 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.041+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.043 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.041+0000 7f972780a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.043 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.041+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.043 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.042+0000 7f972780a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.054 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.052+0000 7f972780a500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:37:03.054 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.052+0000 7f972780a500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:37:03.054 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.052+0000 7f972780a500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:37:03.054 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.052+0000 7f972780a500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:37:03.054 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.052+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.055 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.053+0000 7f972780a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.055 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.053+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.058 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.056+0000 7f972780a500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:37:03.058 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.056+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.058 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.057+0000 7f972780a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.058 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.057+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.059 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.057+0000 7f972780a500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:37:03.059 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.057+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.059 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.057+0000 7f972780a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.059 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.057+0000 7f972780a500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:37:03.059 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.057+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.059 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.057+0000 7f972780a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.060 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.057+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.060 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.058+0000 7f972780a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.060 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.058+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.061 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.058+0000 7f972780a500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:37:03.061 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.058+0000 7f972780a500 20 zonegroup default 2023-09-22T13:37:03.061 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.058+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.061 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.059+0000 7f972780a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.061 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.059+0000 7f972780a500 10 Cannot find current period zone using local zone 2023-09-22T13:37:03.061 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.059+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.061 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.059+0000 7f972780a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.061 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.059+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.062 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.059+0000 7f972780a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.062 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.059+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.062 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.060+0000 7f972780a500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:37:03.062 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.060+0000 7f972780a500 20 zone default found 2023-09-22T13:37:03.062 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.060+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.062 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.060+0000 7f972780a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.063 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.060+0000 7f972780a500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.063 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.060+0000 7f972780a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.063 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.060+0000 7f972780a500 20 started sync module instance, tier type = 2023-09-22T13:37:03.063 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.060+0000 7f972780a500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:37:03.079 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.077+0000 7f972780a500 20 add_watcher() i=0 2023-09-22T13:37:03.079 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.077+0000 7f972780a500 20 add_watcher() i=1 2023-09-22T13:37:03.079 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.077+0000 7f972780a500 20 add_watcher() i=2 2023-09-22T13:37:03.079 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.077+0000 7f972780a500 20 add_watcher() i=3 2023-09-22T13:37:03.079 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.077+0000 7f972780a500 20 add_watcher() i=4 2023-09-22T13:37:03.079 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.077+0000 7f972780a500 20 add_watcher() i=5 2023-09-22T13:37:03.079 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.077+0000 7f972780a500 20 add_watcher() i=6 2023-09-22T13:37:03.081 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.079+0000 7f972780a500 20 add_watcher() i=7 2023-09-22T13:37:03.081 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.079+0000 7f972780a500 2 all 8 watchers are set, enabling cache 2023-09-22T13:37:03.081 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.080+0000 7f97134ed700 0 -- 172.21.15.69:0/3372479610 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x5623ce8818d0 msgr2=0x5623ce883cc0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:37:03.083 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.082+0000 7f965ffff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:37:03.083 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.082+0000 7f972780a500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:03.083 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.082+0000 7f972780a500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:03.086 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.085+0000 7f97134ed700 0 -- 172.21.15.69:0/3014762033 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x5623ce8a10e0 msgr2=0x5623ce8a7b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:37:03.088 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.087+0000 7f972780a500 5 note: GC not initialized 2023-09-22T13:37:03.090 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.088+0000 7f9656ffd700 20 reqs_thread_entry: start 2023-09-22T13:37:03.090 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.088+0000 7f9654ff9700 20 reqs_thread_entry: start 2023-09-22T13:37:03.090 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.088+0000 7f972780a500 20 init_complete bucket index max shards: 11 2023-09-22T13:37:03.090 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.088+0000 7f972780a500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:37:03.090 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.088+0000 7f9642ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:37:33 2023 2023-09-22T13:37:03.090 INFO:teuthology.orchestra.run.smithi069.stderr: 2023-09-22T13:37:03.091 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.088+0000 7f96427fc700 20 reqs_thread_entry: start 2023-09-22T13:37:03.099 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.097+0000 7f972780a500 20 remove_watcher() i=0 2023-09-22T13:37:03.099 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.097+0000 7f972780a500 2 removed watcher, disabling cache 2023-09-22T13:37:03.103 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.102+0000 7f972780a500 20 remove_watcher() i=1 2023-09-22T13:37:03.104 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.103+0000 7f972780a500 20 remove_watcher() i=2 2023-09-22T13:37:03.105 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.104+0000 7f972780a500 20 remove_watcher() i=3 2023-09-22T13:37:03.106 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.105+0000 7f972780a500 20 remove_watcher() i=4 2023-09-22T13:37:03.107 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.106+0000 7f972780a500 20 remove_watcher() i=5 2023-09-22T13:37:03.108 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.107+0000 7f972780a500 20 remove_watcher() i=6 2023-09-22T13:37:03.109 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.108+0000 7f972780a500 20 remove_watcher() i=7 2023-09-22T13:37:03.110 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.109+0000 7f965ffff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:37:03.113 INFO:teuthology.orchestra.run.smithi069.stdout:[] 2023-09-22T13:37:03.115 DEBUG:tasks.util.rgw: json result: [] 2023-09-22T13:37:03.115 INFO:tasks.rgw:Configuring storage class type = LUKEWARM 2023-09-22T13:37:03.115 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM'] 2023-09-22T13:37:03.115 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM'] 2023-09-22T13:37:03.115 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zonegroup placement add --rgw-zone default --placement-id default-placement --storage-class LUKEWARM 2023-09-22T13:37:03.201 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:03.201 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:03.207 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.206+0000 7fc4aab0c500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:03.208 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.206+0000 7fc4aab0c500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:03.212 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.210+0000 7fc3df7fe700 20 reqs_thread_entry: start 2023-09-22T13:37:03.212 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.210+0000 7fc4aab0c500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.212 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.211+0000 7fc4aab0c500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.212 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.211+0000 7fc4aab0c500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.213 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.212+0000 7fc4aab0c500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.213 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.212+0000 7fc4aab0c500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.214 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.212+0000 7fc4aab0c500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:37:03.214 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.212+0000 7fc4aab0c500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.214 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.213+0000 7fc4aab0c500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:37:03.218 INFO:teuthology.orchestra.run.smithi069.stdout:[ 2023-09-22T13:37:03.218 INFO:teuthology.orchestra.run.smithi069.stdout: { 2023-09-22T13:37:03.218 INFO:teuthology.orchestra.run.smithi069.stdout: "key": "default-placement", 2023-09-22T13:37:03.218 INFO:teuthology.orchestra.run.smithi069.stdout: "val": { 2023-09-22T13:37:03.219 INFO:teuthology.orchestra.run.smithi069.stdout: "name": "default-placement", 2023-09-22T13:37:03.219 INFO:teuthology.orchestra.run.smithi069.stdout: "tags": [], 2023-09-22T13:37:03.219 INFO:teuthology.orchestra.run.smithi069.stdout: "storage_classes": [ 2023-09-22T13:37:03.219 INFO:teuthology.orchestra.run.smithi069.stdout: "LUKEWARM", 2023-09-22T13:37:03.219 INFO:teuthology.orchestra.run.smithi069.stdout: "STANDARD" 2023-09-22T13:37:03.219 INFO:teuthology.orchestra.run.smithi069.stdout: ] 2023-09-22T13:37:03.219 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:03.219 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:03.219 INFO:teuthology.orchestra.run.smithi069.stdout:] 2023-09-22T13:37:03.220 DEBUG:tasks.util.rgw: json result: [{'key': 'default-placement', 'val': {'name': 'default-placement', 'tags': [], 'storage_classes': ['LUKEWARM', 'STANDARD']}}] 2023-09-22T13:37:03.220 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM', '--data-pool', 'default.rgw.buckets.data.lukewarm'] 2023-09-22T13:37:03.220 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM', '--data-pool', 'default.rgw.buckets.data.lukewarm'] 2023-09-22T13:37:03.220 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zone placement add --rgw-zone default --placement-id default-placement --storage-class LUKEWARM --data-pool default.rgw.buckets.data.lukewarm 2023-09-22T13:37:03.268 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:03.268 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:03.274 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.272+0000 7ff11ff4c500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:03.274 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.272+0000 7ff11ff4c500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:03.279 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.277+0000 7ff04ffff700 20 reqs_thread_entry: start 2023-09-22T13:37:03.279 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.277+0000 7ff11ff4c500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.279 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.278+0000 7ff11ff4c500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.280 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.278+0000 7ff11ff4c500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.280 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.279+0000 7ff11ff4c500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:37:03.280 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.279+0000 7ff11ff4c500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.281 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.279+0000 7ff11ff4c500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.281 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.279+0000 7ff11ff4c500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.281 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.280+0000 7ff11ff4c500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.282 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.280+0000 7ff11ff4c500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.282 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.280+0000 7ff11ff4c500 20 rados_obj.operate() r=0 bl.length=382 2023-09-22T13:37:03.286 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2023-09-22T13:37:03.286 INFO:teuthology.orchestra.run.smithi069.stdout: "id": "f54d190c-30d0-4b87-88c4-5bcc7de96de4", 2023-09-22T13:37:03.286 INFO:teuthology.orchestra.run.smithi069.stdout: "name": "default", 2023-09-22T13:37:03.286 INFO:teuthology.orchestra.run.smithi069.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:37:03.286 INFO:teuthology.orchestra.run.smithi069.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:37:03.286 INFO:teuthology.orchestra.run.smithi069.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:37:03.286 INFO:teuthology.orchestra.run.smithi069.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:37:03.286 INFO:teuthology.orchestra.run.smithi069.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "system_key": { 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "access_key": "", 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: "secret_key": "" 2023-09-22T13:37:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: "placement_pools": [ 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: { 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: "key": "default-placement", 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: "val": { 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: "storage_classes": { 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: "LUKEWARM": { 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: "data_pool": "default.rgw.buckets.data.lukewarm" 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: "STANDARD": { 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: "compression_type": "random" 2023-09-22T13:37:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:03.289 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:37:03.289 INFO:teuthology.orchestra.run.smithi069.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:37:03.289 INFO:teuthology.orchestra.run.smithi069.stdout: "index_type": 0 2023-09-22T13:37:03.289 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:03.289 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:03.289 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2023-09-22T13:37:03.289 INFO:teuthology.orchestra.run.smithi069.stdout: "realm_id": "", 2023-09-22T13:37:03.289 INFO:teuthology.orchestra.run.smithi069.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:37:03.289 INFO:teuthology.orchestra.run.smithi069.stdout:} 2023-09-22T13:37:03.290 DEBUG:tasks.util.rgw: json result: {'id': 'f54d190c-30d0-4b87-88c4-5bcc7de96de4', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'LUKEWARM': {'data_pool': 'default.rgw.buckets.data.lukewarm'}, 'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:37:03.290 INFO:tasks.rgw:Configuring storage class type = FROZEN 2023-09-22T13:37:03.290 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN'] 2023-09-22T13:37:03.290 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN'] 2023-09-22T13:37:03.290 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zonegroup placement add --rgw-zone default --placement-id default-placement --storage-class FROZEN 2023-09-22T13:37:03.335 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:03.336 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:03.341 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.340+0000 7f8188875500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:03.342 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.340+0000 7f8188875500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:03.346 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.344+0000 7f80c57fa700 20 reqs_thread_entry: start 2023-09-22T13:37:03.346 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.344+0000 7f8188875500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.346 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.345+0000 7f8188875500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.347 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.345+0000 7f8188875500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.347 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.346+0000 7f8188875500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.347 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.346+0000 7f8188875500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.348 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.346+0000 7f8188875500 20 rados_obj.operate() r=0 bl.length=382 2023-09-22T13:37:03.348 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.346+0000 7f8188875500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.349 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.347+0000 7f8188875500 20 rados_obj.operate() r=0 bl.length=956 2023-09-22T13:37:03.352 INFO:teuthology.orchestra.run.smithi069.stdout:[ 2023-09-22T13:37:03.352 INFO:teuthology.orchestra.run.smithi069.stdout: { 2023-09-22T13:37:03.352 INFO:teuthology.orchestra.run.smithi069.stdout: "key": "default-placement", 2023-09-22T13:37:03.352 INFO:teuthology.orchestra.run.smithi069.stdout: "val": { 2023-09-22T13:37:03.353 INFO:teuthology.orchestra.run.smithi069.stdout: "name": "default-placement", 2023-09-22T13:37:03.353 INFO:teuthology.orchestra.run.smithi069.stdout: "tags": [], 2023-09-22T13:37:03.353 INFO:teuthology.orchestra.run.smithi069.stdout: "storage_classes": [ 2023-09-22T13:37:03.353 INFO:teuthology.orchestra.run.smithi069.stdout: "FROZEN", 2023-09-22T13:37:03.353 INFO:teuthology.orchestra.run.smithi069.stdout: "LUKEWARM", 2023-09-22T13:37:03.353 INFO:teuthology.orchestra.run.smithi069.stdout: "STANDARD" 2023-09-22T13:37:03.353 INFO:teuthology.orchestra.run.smithi069.stdout: ] 2023-09-22T13:37:03.353 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:03.353 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:03.353 INFO:teuthology.orchestra.run.smithi069.stdout:] 2023-09-22T13:37:03.354 DEBUG:tasks.util.rgw: json result: [{'key': 'default-placement', 'val': {'name': 'default-placement', 'tags': [], 'storage_classes': ['FROZEN', 'LUKEWARM', 'STANDARD']}}] 2023-09-22T13:37:03.354 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN', '--data-pool', 'default.rgw.buckets.data.frozen'] 2023-09-22T13:37:03.354 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN', '--data-pool', 'default.rgw.buckets.data.frozen'] 2023-09-22T13:37:03.355 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zone placement add --rgw-zone default --placement-id default-placement --storage-class FROZEN --data-pool default.rgw.buckets.data.frozen 2023-09-22T13:37:03.401 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:03.401 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:03.408 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.406+0000 7fbddd956500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:03.408 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.406+0000 7fbddd956500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:03.413 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.412+0000 7fbd127fc700 20 reqs_thread_entry: start 2023-09-22T13:37:03.413 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.412+0000 7fbddd956500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.414 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.412+0000 7fbddd956500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.414 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.412+0000 7fbddd956500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.415 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.413+0000 7fbddd956500 20 rados_obj.operate() r=0 bl.length=956 2023-09-22T13:37:03.415 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.413+0000 7fbddd956500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.415 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.414+0000 7fbddd956500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.415 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.414+0000 7fbddd956500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.416 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.414+0000 7fbddd956500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.416 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.414+0000 7fbddd956500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.417 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:37:03.416+0000 7fbddd956500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:37:03.421 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2023-09-22T13:37:03.421 INFO:teuthology.orchestra.run.smithi069.stdout: "id": "f54d190c-30d0-4b87-88c4-5bcc7de96de4", 2023-09-22T13:37:03.421 INFO:teuthology.orchestra.run.smithi069.stdout: "name": "default", 2023-09-22T13:37:03.421 INFO:teuthology.orchestra.run.smithi069.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:37:03.421 INFO:teuthology.orchestra.run.smithi069.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:37:03.421 INFO:teuthology.orchestra.run.smithi069.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:37:03.421 INFO:teuthology.orchestra.run.smithi069.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:37:03.421 INFO:teuthology.orchestra.run.smithi069.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "system_key": { 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "access_key": "", 2023-09-22T13:37:03.422 INFO:teuthology.orchestra.run.smithi069.stdout: "secret_key": "" 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: "placement_pools": [ 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: { 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: "key": "default-placement", 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: "val": { 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: "storage_classes": { 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: "FROZEN": { 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: "data_pool": "default.rgw.buckets.data.frozen" 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: "LUKEWARM": { 2023-09-22T13:37:03.423 INFO:teuthology.orchestra.run.smithi069.stdout: "data_pool": "default.rgw.buckets.data.lukewarm" 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: "STANDARD": { 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: "compression_type": "random" 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: "index_type": 0 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2023-09-22T13:37:03.424 INFO:teuthology.orchestra.run.smithi069.stdout: "realm_id": "", 2023-09-22T13:37:03.425 INFO:teuthology.orchestra.run.smithi069.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:37:03.425 INFO:teuthology.orchestra.run.smithi069.stdout:} 2023-09-22T13:37:03.425 DEBUG:tasks.util.rgw: json result: {'id': 'f54d190c-30d0-4b87-88c4-5bcc7de96de4', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'FROZEN': {'data_pool': 'default.rgw.buckets.data.frozen'}, 'LUKEWARM': {'data_pool': 'default.rgw.buckets.data.lukewarm'}, 'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:37:03.426 INFO:tasks.util.rgw:rgwadmin: client.1 : ['user', 'list'] 2023-09-22T13:37:03.426 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'user', 'list'] 2023-09-22T13:37:03.426 DEBUG:teuthology.orchestra.run.smithi174:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph user list 2023-09-22T13:37:03.465 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:03.465 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:03.471 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.471+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.473 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.473+0000 7ff82ac00500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.474 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.473+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.474 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.473+0000 7ff82ac00500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.474 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.473+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.475 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.474+0000 7ff82ac00500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.475 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.474+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.475 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.474+0000 7ff82ac00500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.486 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.484+0000 7ff82ac00500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:37:03.486 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.484+0000 7ff82ac00500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:37:03.486 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.484+0000 7ff82ac00500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:37:03.486 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.484+0000 7ff82ac00500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:37:03.486 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.484+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.486 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.485+0000 7ff82ac00500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.486 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.485+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.487 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.485+0000 7ff82ac00500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:37:03.487 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.485+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.488 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.485+0000 7ff82ac00500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.488 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.485+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.488 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.486+0000 7ff82ac00500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:37:03.488 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.486+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.488 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.486+0000 7ff82ac00500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.488 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.486+0000 7ff82ac00500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:37:03.488 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.486+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.488 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.486+0000 7ff82ac00500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.488 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.486+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.488 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.487+0000 7ff82ac00500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.488 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.487+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.490 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.487+0000 7ff82ac00500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:37:03.490 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.487+0000 7ff82ac00500 20 zonegroup default 2023-09-22T13:37:03.490 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.487+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.490 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.487+0000 7ff82ac00500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.490 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.487+0000 7ff82ac00500 10 Cannot find current period zone using local zone 2023-09-22T13:37:03.490 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.487+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.487+0000 7ff82ac00500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.487+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.488+0000 7ff82ac00500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.488+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.488+0000 7ff82ac00500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.488+0000 7ff82ac00500 20 zone default found 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.488+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.488+0000 7ff82ac00500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.488+0000 7ff82ac00500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.488+0000 7ff82ac00500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.488+0000 7ff82ac00500 20 started sync module instance, tier type = 2023-09-22T13:37:03.491 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.488+0000 7ff82ac00500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:37:03.499 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.498+0000 7ff8160e2700 0 -- 172.21.15.174:0/943681743 >> [v2:172.21.15.69:6808/145166,v1:172.21.15.69:6809/145166] conn(0x561e343bbc70 msgr2=0x561e343c1ca0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:37:03.500 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.499+0000 7ff82ac00500 20 add_watcher() i=0 2023-09-22T13:37:03.501 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.499+0000 7ff82ac00500 20 add_watcher() i=1 2023-09-22T13:37:03.501 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.499+0000 7ff82ac00500 20 add_watcher() i=2 2023-09-22T13:37:03.501 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.499+0000 7ff82ac00500 20 add_watcher() i=3 2023-09-22T13:37:03.501 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.499+0000 7ff82ac00500 20 add_watcher() i=4 2023-09-22T13:37:03.501 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.500+0000 7ff82ac00500 20 add_watcher() i=5 2023-09-22T13:37:03.501 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.500+0000 7ff82ac00500 20 add_watcher() i=6 2023-09-22T13:37:03.501 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.501+0000 7ff82ac00500 20 add_watcher() i=7 2023-09-22T13:37:03.502 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.501+0000 7ff82ac00500 2 all 8 watchers are set, enabling cache 2023-09-22T13:37:03.503 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.502+0000 7ff7e7fdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:37:03.503 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.502+0000 7ff82ac00500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:03.503 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.502+0000 7ff82ac00500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:03.505 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.505+0000 7ff8158e1700 0 -- 172.21.15.174:0/3295368340 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x561e343ddee0 msgr2=0x561e343e3b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:37:03.507 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.507+0000 7ff82ac00500 5 note: GC not initialized 2023-09-22T13:37:03.509 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.508+0000 7ff756ffd700 20 reqs_thread_entry: start 2023-09-22T13:37:03.509 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.508+0000 7ff754ff9700 20 reqs_thread_entry: start 2023-09-22T13:37:03.509 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.508+0000 7ff82ac00500 20 init_complete bucket index max shards: 11 2023-09-22T13:37:03.509 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.508+0000 7ff82ac00500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:37:03.509 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.508+0000 7ff74affd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:37:33 2023 2023-09-22T13:37:03.509 INFO:teuthology.orchestra.run.smithi174.stderr: 2023-09-22T13:37:03.510 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.508+0000 7ff74a7fc700 20 reqs_thread_entry: start 2023-09-22T13:37:03.518 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.518+0000 7ff82ac00500 20 remove_watcher() i=0 2023-09-22T13:37:03.519 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.518+0000 7ff82ac00500 2 removed watcher, disabling cache 2023-09-22T13:37:03.520 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.519+0000 7ff82ac00500 20 remove_watcher() i=1 2023-09-22T13:37:03.520 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.520+0000 7ff82ac00500 20 remove_watcher() i=2 2023-09-22T13:37:03.522 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.521+0000 7ff82ac00500 20 remove_watcher() i=3 2023-09-22T13:37:03.522 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.522+0000 7ff82ac00500 20 remove_watcher() i=4 2023-09-22T13:37:03.524 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.523+0000 7ff82ac00500 20 remove_watcher() i=5 2023-09-22T13:37:03.524 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.524+0000 7ff82ac00500 20 remove_watcher() i=6 2023-09-22T13:37:03.525 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.525+0000 7ff82ac00500 20 remove_watcher() i=7 2023-09-22T13:37:03.526 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.526+0000 7ff7e7fdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:37:03.529 INFO:teuthology.orchestra.run.smithi174.stdout:[] 2023-09-22T13:37:03.531 DEBUG:tasks.util.rgw: json result: [] 2023-09-22T13:37:03.531 INFO:tasks.rgw:Configuring storage class type = LUKEWARM 2023-09-22T13:37:03.531 INFO:tasks.util.rgw:rgwadmin: client.1 : ['zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM'] 2023-09-22T13:37:03.531 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM'] 2023-09-22T13:37:03.531 DEBUG:teuthology.orchestra.run.smithi174:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph zonegroup placement add --rgw-zone default --placement-id default-placement --storage-class LUKEWARM 2023-09-22T13:37:03.617 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:03.618 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:03.623 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.623+0000 7f1f459f4500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:03.624 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.623+0000 7f1f459f4500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:03.627 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.627+0000 7f1e79ffb700 20 reqs_thread_entry: start 2023-09-22T13:37:03.627 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.627+0000 7f1f459f4500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.628 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.628+0000 7f1f459f4500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.628 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.628+0000 7f1f459f4500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.629 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.628+0000 7f1f459f4500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.629 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.628+0000 7f1f459f4500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.630 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.629+0000 7f1f459f4500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:37:03.630 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.629+0000 7f1f459f4500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.630 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.630+0000 7f1f459f4500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:37:03.634 INFO:teuthology.orchestra.run.smithi174.stdout:[ 2023-09-22T13:37:03.634 INFO:teuthology.orchestra.run.smithi174.stdout: { 2023-09-22T13:37:03.634 INFO:teuthology.orchestra.run.smithi174.stdout: "key": "default-placement", 2023-09-22T13:37:03.635 INFO:teuthology.orchestra.run.smithi174.stdout: "val": { 2023-09-22T13:37:03.635 INFO:teuthology.orchestra.run.smithi174.stdout: "name": "default-placement", 2023-09-22T13:37:03.635 INFO:teuthology.orchestra.run.smithi174.stdout: "tags": [], 2023-09-22T13:37:03.635 INFO:teuthology.orchestra.run.smithi174.stdout: "storage_classes": [ 2023-09-22T13:37:03.635 INFO:teuthology.orchestra.run.smithi174.stdout: "FROZEN", 2023-09-22T13:37:03.635 INFO:teuthology.orchestra.run.smithi174.stdout: "LUKEWARM", 2023-09-22T13:37:03.635 INFO:teuthology.orchestra.run.smithi174.stdout: "STANDARD" 2023-09-22T13:37:03.635 INFO:teuthology.orchestra.run.smithi174.stdout: ] 2023-09-22T13:37:03.635 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:03.635 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:03.635 INFO:teuthology.orchestra.run.smithi174.stdout:] 2023-09-22T13:37:03.636 DEBUG:tasks.util.rgw: json result: [{'key': 'default-placement', 'val': {'name': 'default-placement', 'tags': [], 'storage_classes': ['FROZEN', 'LUKEWARM', 'STANDARD']}}] 2023-09-22T13:37:03.636 INFO:tasks.util.rgw:rgwadmin: client.1 : ['zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM', '--data-pool', 'default.rgw.buckets.data.lukewarm'] 2023-09-22T13:37:03.636 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM', '--data-pool', 'default.rgw.buckets.data.lukewarm'] 2023-09-22T13:37:03.637 DEBUG:teuthology.orchestra.run.smithi174:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph zone placement add --rgw-zone default --placement-id default-placement --storage-class LUKEWARM --data-pool default.rgw.buckets.data.lukewarm 2023-09-22T13:37:03.684 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:03.844 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:03.845 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.689+0000 7f3fe449b500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:03.845 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.689+0000 7f3fe449b500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:03.845 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.693+0000 7f3f497fa700 20 reqs_thread_entry: start 2023-09-22T13:37:03.845 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.693+0000 7f3fe449b500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.845 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.694+0000 7f3fe449b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.846 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.694+0000 7f3fe449b500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.846 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.694+0000 7f3fe449b500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:37:03.846 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.694+0000 7f3fe449b500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.846 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.695+0000 7f3fe449b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.846 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.695+0000 7f3fe449b500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.846 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.695+0000 7f3fe449b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.846 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.695+0000 7f3fe449b500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.846 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.697+0000 7f3fe449b500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:37:03.847 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2023-09-22T13:37:03.847 INFO:teuthology.orchestra.run.smithi174.stdout: "id": "f54d190c-30d0-4b87-88c4-5bcc7de96de4", 2023-09-22T13:37:03.847 INFO:teuthology.orchestra.run.smithi174.stdout: "name": "default", 2023-09-22T13:37:03.847 INFO:teuthology.orchestra.run.smithi174.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:37:03.847 INFO:teuthology.orchestra.run.smithi174.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:37:03.848 INFO:teuthology.orchestra.run.smithi174.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:37:03.848 INFO:teuthology.orchestra.run.smithi174.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:37:03.848 INFO:teuthology.orchestra.run.smithi174.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:37:03.848 INFO:teuthology.orchestra.run.smithi174.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:37:03.848 INFO:teuthology.orchestra.run.smithi174.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:37:03.848 INFO:teuthology.orchestra.run.smithi174.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:37:03.848 INFO:teuthology.orchestra.run.smithi174.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:37:03.848 INFO:teuthology.orchestra.run.smithi174.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:37:03.848 INFO:teuthology.orchestra.run.smithi174.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:37:03.848 INFO:teuthology.orchestra.run.smithi174.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:37:03.848 INFO:teuthology.orchestra.run.smithi174.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: "system_key": { 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: "access_key": "", 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: "secret_key": "" 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: "placement_pools": [ 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: { 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: "key": "default-placement", 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: "val": { 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: "storage_classes": { 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: "FROZEN": { 2023-09-22T13:37:03.849 INFO:teuthology.orchestra.run.smithi174.stdout: "data_pool": "default.rgw.buckets.data.frozen" 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: "LUKEWARM": { 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: "data_pool": "default.rgw.buckets.data.lukewarm" 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: "STANDARD": { 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: "compression_type": "random" 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: "index_type": 0 2023-09-22T13:37:03.850 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:03.851 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:03.851 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2023-09-22T13:37:03.851 INFO:teuthology.orchestra.run.smithi174.stdout: "realm_id": "", 2023-09-22T13:37:03.851 INFO:teuthology.orchestra.run.smithi174.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:37:03.851 INFO:teuthology.orchestra.run.smithi174.stdout:} 2023-09-22T13:37:03.851 DEBUG:tasks.util.rgw: json result: {'id': 'f54d190c-30d0-4b87-88c4-5bcc7de96de4', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'FROZEN': {'data_pool': 'default.rgw.buckets.data.frozen'}, 'LUKEWARM': {'data_pool': 'default.rgw.buckets.data.lukewarm'}, 'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:37:03.888 INFO:tasks.rgw:Configuring storage class type = FROZEN 2023-09-22T13:37:03.888 INFO:tasks.util.rgw:rgwadmin: client.1 : ['zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN'] 2023-09-22T13:37:03.888 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN'] 2023-09-22T13:37:03.888 DEBUG:teuthology.orchestra.run.smithi174:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph zonegroup placement add --rgw-zone default --placement-id default-placement --storage-class FROZEN 2023-09-22T13:37:03.934 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:03.935 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:03.940 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.940+0000 7f27eec47500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:03.941 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.940+0000 7f27eec47500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:03.944 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.943+0000 7f2723fff700 20 reqs_thread_entry: start 2023-09-22T13:37:03.944 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.943+0000 7f27eec47500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.945 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.944+0000 7f27eec47500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:03.945 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.944+0000 7f27eec47500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.946 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.945+0000 7f27eec47500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:03.946 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.945+0000 7f27eec47500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.946 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.946+0000 7f27eec47500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:37:03.947 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.946+0000 7f27eec47500 20 rados->read ofs=0 len=0 2023-09-22T13:37:03.947 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:03.946+0000 7f27eec47500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:37:03.951 INFO:teuthology.orchestra.run.smithi174.stdout:[ 2023-09-22T13:37:03.951 INFO:teuthology.orchestra.run.smithi174.stdout: { 2023-09-22T13:37:03.951 INFO:teuthology.orchestra.run.smithi174.stdout: "key": "default-placement", 2023-09-22T13:37:03.951 INFO:teuthology.orchestra.run.smithi174.stdout: "val": { 2023-09-22T13:37:03.951 INFO:teuthology.orchestra.run.smithi174.stdout: "name": "default-placement", 2023-09-22T13:37:03.951 INFO:teuthology.orchestra.run.smithi174.stdout: "tags": [], 2023-09-22T13:37:03.951 INFO:teuthology.orchestra.run.smithi174.stdout: "storage_classes": [ 2023-09-22T13:37:03.952 INFO:teuthology.orchestra.run.smithi174.stdout: "FROZEN", 2023-09-22T13:37:03.952 INFO:teuthology.orchestra.run.smithi174.stdout: "LUKEWARM", 2023-09-22T13:37:03.952 INFO:teuthology.orchestra.run.smithi174.stdout: "STANDARD" 2023-09-22T13:37:03.952 INFO:teuthology.orchestra.run.smithi174.stdout: ] 2023-09-22T13:37:03.952 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:03.952 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:03.952 INFO:teuthology.orchestra.run.smithi174.stdout:] 2023-09-22T13:37:03.953 DEBUG:tasks.util.rgw: json result: [{'key': 'default-placement', 'val': {'name': 'default-placement', 'tags': [], 'storage_classes': ['FROZEN', 'LUKEWARM', 'STANDARD']}}] 2023-09-22T13:37:03.953 INFO:tasks.util.rgw:rgwadmin: client.1 : ['zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN', '--data-pool', 'default.rgw.buckets.data.frozen'] 2023-09-22T13:37:03.953 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN', '--data-pool', 'default.rgw.buckets.data.frozen'] 2023-09-22T13:37:03.953 DEBUG:teuthology.orchestra.run.smithi174:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph zone placement add --rgw-zone default --placement-id default-placement --storage-class FROZEN --data-pool default.rgw.buckets.data.frozen 2023-09-22T13:37:04.000 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:37:04.000 INFO:teuthology.orchestra.run.smithi174.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:37:04.006 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.005+0000 7fcd2ed0e500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:37:04.006 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.005+0000 7fcd2ed0e500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:37:04.010 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.009+0000 7fcc63fff700 20 reqs_thread_entry: start 2023-09-22T13:37:04.010 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.009+0000 7fcd2ed0e500 20 rados->read ofs=0 len=0 2023-09-22T13:37:04.011 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.010+0000 7fcd2ed0e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:04.011 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.010+0000 7fcd2ed0e500 20 rados->read ofs=0 len=0 2023-09-22T13:37:04.011 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.011+0000 7fcd2ed0e500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:37:04.011 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.011+0000 7fcd2ed0e500 20 rados->read ofs=0 len=0 2023-09-22T13:37:04.012 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.011+0000 7fcd2ed0e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:37:04.012 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.011+0000 7fcd2ed0e500 20 rados->read ofs=0 len=0 2023-09-22T13:37:04.012 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.011+0000 7fcd2ed0e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:37:04.012 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.011+0000 7fcd2ed0e500 20 rados->read ofs=0 len=0 2023-09-22T13:37:04.014 INFO:teuthology.orchestra.run.smithi174.stderr:2023-09-22T13:37:04.013+0000 7fcd2ed0e500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:37:04.018 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2023-09-22T13:37:04.018 INFO:teuthology.orchestra.run.smithi174.stdout: "id": "f54d190c-30d0-4b87-88c4-5bcc7de96de4", 2023-09-22T13:37:04.018 INFO:teuthology.orchestra.run.smithi174.stdout: "name": "default", 2023-09-22T13:37:04.018 INFO:teuthology.orchestra.run.smithi174.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:37:04.018 INFO:teuthology.orchestra.run.smithi174.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:37:04.018 INFO:teuthology.orchestra.run.smithi174.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:37:04.018 INFO:teuthology.orchestra.run.smithi174.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:37:04.018 INFO:teuthology.orchestra.run.smithi174.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "system_key": { 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "access_key": "", 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: "secret_key": "" 2023-09-22T13:37:04.019 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: "placement_pools": [ 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: { 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: "key": "default-placement", 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: "val": { 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: "storage_classes": { 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: "FROZEN": { 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: "data_pool": "default.rgw.buckets.data.frozen" 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: "LUKEWARM": { 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: "data_pool": "default.rgw.buckets.data.lukewarm" 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2023-09-22T13:37:04.020 INFO:teuthology.orchestra.run.smithi174.stdout: "STANDARD": { 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout: "compression_type": "random" 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout: }, 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout: "index_type": 0 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout: } 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout: ], 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout: "realm_id": "", 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:37:04.021 INFO:teuthology.orchestra.run.smithi174.stdout:} 2023-09-22T13:37:04.022 DEBUG:tasks.util.rgw: json result: {'id': 'f54d190c-30d0-4b87-88c4-5bcc7de96de4', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'FROZEN': {'data_pool': 'default.rgw.buckets.data.frozen'}, 'LUKEWARM': {'data_pool': 'default.rgw.buckets.data.lukewarm'}, 'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:37:04.022 INFO:tasks.rgw:Starting rgw... 2023-09-22T13:37:04.022 INFO:tasks.rgw:rgw client.0 config is {'ssl certificate': 'rgw.client.0'} 2023-09-22T13:37:04.023 INFO:tasks.rgw:Using civetweb as radosgw frontend 2023-09-22T13:37:04.023 INFO:tasks.rgw.client.0:Restarting daemon 2023-09-22T13:37:04.023 DEBUG:teuthology.orchestra.run.smithi069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'civetweb ssl_certificate=/home/ubuntu/cephtest/ca/rgw.client.0.crt port=443s' -n client.0 --cluster ceph -k /etc/ceph/ceph.client.0.keyring --log-file /var/log/ceph/rgw.ceph.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.ceph.client.0.sock --foreground | sudo tee /var/log/ceph/rgw.ceph.client.0.stdout 2>&1 2023-09-22T13:37:04.025 INFO:tasks.rgw.client.0:Started 2023-09-22T13:37:04.025 INFO:tasks.rgw:rgw client.1 config is {'ssl certificate': 'rgw.client.1'} 2023-09-22T13:37:04.025 INFO:tasks.rgw:Using civetweb as radosgw frontend 2023-09-22T13:37:04.026 INFO:tasks.rgw.client.1:Restarting daemon 2023-09-22T13:37:04.026 DEBUG:teuthology.orchestra.run.smithi174:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'civetweb ssl_certificate=/home/ubuntu/cephtest/ca/rgw.client.1.crt port=443s' -n client.1 --cluster ceph -k /etc/ceph/ceph.client.1.keyring --log-file /var/log/ceph/rgw.ceph.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.ceph.client.1.sock --foreground | sudo tee /var/log/ceph/rgw.ceph.client.1.stdout 2>&1 2023-09-22T13:37:04.028 INFO:tasks.rgw.client.1:Started 2023-09-22T13:37:04.028 INFO:tasks.rgw:Polling client.0 until it starts accepting connections on https://smithi069.front.sepia.ceph.com:443/ 2023-09-22T13:37:04.028 DEBUG:teuthology.orchestra.run.smithi069:> curl https://smithi069.front.sepia.ceph.com:443/ 2023-09-22T13:37:04.066 INFO:teuthology.orchestra.run.smithi069.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2023-09-22T13:37:04.066 INFO:teuthology.orchestra.run.smithi069.stderr: Dload Upload Total Spent Left Speed 2023-09-22T13:37:04.068 DEBUG:teuthology.orchestra.run:got remote process result: 7 2023-09-22T13:37:04.068 INFO:teuthology.orchestra.run.smithi069.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi069.front.sepia.ceph.com port 443: Connection refused 2023-09-22T13:37:04.228 INFO:tasks.rgw.client.0.smithi069.stdout:2023-09-22T13:37:04.227+0000 7fcd046bd5c0 -1 IMPORTANT: the civetweb frontend is now deprecated and will be removed in a future release 2023-09-22T13:37:04.233 INFO:tasks.rgw.client.1.smithi174.stdout:2023-09-22T13:37:04.232+0000 7f16e7f135c0 -1 IMPORTANT: the civetweb frontend is now deprecated and will be removed in a future release 2023-09-22T13:37:05.069 DEBUG:teuthology.orchestra.run.smithi069:> curl https://smithi069.front.sepia.ceph.com:443/ 2023-09-22T13:37:05.091 INFO:teuthology.orchestra.run.smithi069.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2023-09-22T13:37:05.092 INFO:teuthology.orchestra.run.smithi069.stderr: Dload Upload Total Spent Left Speed 2023-09-22T13:37:05.182 INFO:teuthology.orchestra.run.smithi069.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 2377 0 --:--:-- --:--:-- --:--:-- 2377 2023-09-22T13:37:05.183 INFO:teuthology.orchestra.run.smithi069.stdout:anonymous 2023-09-22T13:37:05.183 INFO:tasks.rgw:Polling client.1 until it starts accepting connections on https://smithi174.front.sepia.ceph.com:443/ 2023-09-22T13:37:05.183 DEBUG:teuthology.orchestra.run.smithi174:> curl https://smithi174.front.sepia.ceph.com:443/ 2023-09-22T13:37:05.224 INFO:teuthology.orchestra.run.smithi174.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2023-09-22T13:37:05.225 INFO:teuthology.orchestra.run.smithi174.stderr: Dload Upload Total Spent Left Speed 2023-09-22T13:37:05.315 INFO:teuthology.orchestra.run.smithi174.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 2377 0 --:--:-- --:--:-- --:--:-- 2377 2023-09-22T13:37:05.316 INFO:teuthology.orchestra.run.smithi174.stdout:anonymous 2023-09-22T13:37:05.316 INFO:teuthology.run_tasks:Running task workunit... 2023-09-22T13:37:05.328 INFO:tasks.workunit:Pulling workunits from ref 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:37:05.329 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-22T13:37:05.329 DEBUG:teuthology.orchestra.run.smithi069:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:37:05.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T13:37:05.347 INFO:teuthology.orchestra.run.smithi069.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-22T13:37:05.348 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:37:05.405 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-22T13:37:05.405 DEBUG:teuthology.orchestra.run.smithi069:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-22T13:37:05.463 INFO:tasks.workunit:timeout=3h 2023-09-22T13:37:05.464 INFO:tasks.workunit:cleanup=True 2023-09-22T13:37:05.464 DEBUG:teuthology.orchestra.run.smithi069:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:37:05.522 INFO:tasks.workunit.client.0.smithi069.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-22T13:38:05.663 INFO:tasks.workunit.client.0.smithi069.stderr:Note: switching to '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'. 2023-09-22T13:38:05.663 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:05.663 INFO:tasks.workunit.client.0.smithi069.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2023-09-22T13:38:05.663 INFO:tasks.workunit.client.0.smithi069.stderr:changes and commit them, and you can discard any commits you make in this 2023-09-22T13:38:05.663 INFO:tasks.workunit.client.0.smithi069.stderr:state without impacting any branches by switching back to a branch. 2023-09-22T13:38:05.664 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:05.664 INFO:tasks.workunit.client.0.smithi069.stderr:If you want to create a new branch to retain commits you create, you may 2023-09-22T13:38:05.664 INFO:tasks.workunit.client.0.smithi069.stderr:do so (now or later) by using -c with the switch command. Example: 2023-09-22T13:38:05.664 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:05.664 INFO:tasks.workunit.client.0.smithi069.stderr: git switch -c 2023-09-22T13:38:05.664 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:05.664 INFO:tasks.workunit.client.0.smithi069.stderr:Or undo this operation with: 2023-09-22T13:38:05.664 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:05.665 INFO:tasks.workunit.client.0.smithi069.stderr: git switch - 2023-09-22T13:38:05.665 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:05.665 INFO:tasks.workunit.client.0.smithi069.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2023-09-22T13:38:05.665 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:05.665 INFO:tasks.workunit.client.0.smithi069.stderr:HEAD is now at 31c126b7c1c rgw: radosgw-admin bucket check should only print index entries with --check-objects flag 2023-09-22T13:38:05.669 DEBUG:teuthology.orchestra.run.smithi069:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-22T13:38:05.957 INFO:tasks.workunit.client.0.smithi069.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-22T13:38:05.962 INFO:tasks.workunit.client.0.smithi069.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:38:05.962 INFO:tasks.workunit.client.0.smithi069.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-22T13:38:06.492 INFO:tasks.workunit.client.0.smithi069.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-22T13:38:06.557 INFO:tasks.workunit.client.0.smithi069.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-22T13:38:06.595 INFO:tasks.workunit.client.0.smithi069.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:38:06.597 INFO:tasks.workunit.client.0.smithi069.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:38:06.597 INFO:tasks.workunit.client.0.smithi069.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-22T13:38:06.633 INFO:tasks.workunit.client.0.smithi069.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:38:06.637 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:38:06.637 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-22T13:38:06.654 INFO:tasks.workunit:Running workunits matching rgw/run-bucket-check.sh on client.0... 2023-09-22T13:38:06.655 INFO:tasks.workunit:Running workunit rgw/run-bucket-check.sh... 2023-09-22T13:38:06.656 DEBUG:teuthology.orchestra.run.smithi069:workunit test rgw/run-bucket-check.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/run-bucket-check.sh 2023-09-22T13:38:06.717 INFO:tasks.workunit.client.0.smithi069.stderr:++ dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/run-bucket-check.sh 2023-09-22T13:38:06.718 INFO:tasks.workunit.client.0.smithi069.stderr:+ mydir=/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw 2023-09-22T13:38:06.718 INFO:tasks.workunit.client.0.smithi069.stderr:+ python3 -m venv /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw 2023-09-22T13:38:08.254 INFO:tasks.workunit.client.0.smithi069.stderr:+ source /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/bin/activate 2023-09-22T13:38:08.254 INFO:tasks.workunit.client.0.smithi069.stderr:++ deactivate nondestructive 2023-09-22T13:38:08.254 INFO:tasks.workunit.client.0.smithi069.stderr:++ '[' -n '' ']' 2023-09-22T13:38:08.254 INFO:tasks.workunit.client.0.smithi069.stderr:++ '[' -n '' ']' 2023-09-22T13:38:08.255 INFO:tasks.workunit.client.0.smithi069.stderr:++ '[' -n /usr/bin/bash -o -n '' ']' 2023-09-22T13:38:08.255 INFO:tasks.workunit.client.0.smithi069.stderr:++ hash -r 2023-09-22T13:38:08.255 INFO:tasks.workunit.client.0.smithi069.stderr:++ '[' -n '' ']' 2023-09-22T13:38:08.255 INFO:tasks.workunit.client.0.smithi069.stderr:++ unset VIRTUAL_ENV 2023-09-22T13:38:08.255 INFO:tasks.workunit.client.0.smithi069.stderr:++ '[' '!' nondestructive = nondestructive ']' 2023-09-22T13:38:08.255 INFO:tasks.workunit.client.0.smithi069.stderr:++ VIRTUAL_ENV=/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw 2023-09-22T13:38:08.255 INFO:tasks.workunit.client.0.smithi069.stderr:++ export VIRTUAL_ENV 2023-09-22T13:38:08.255 INFO:tasks.workunit.client.0.smithi069.stderr:++ _OLD_VIRTUAL_PATH=/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin 2023-09-22T13:38:08.256 INFO:tasks.workunit.client.0.smithi069.stderr:++ PATH=/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/bin:/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin 2023-09-22T13:38:08.256 INFO:tasks.workunit.client.0.smithi069.stderr:++ export PATH 2023-09-22T13:38:08.256 INFO:tasks.workunit.client.0.smithi069.stderr:++ '[' -n '' ']' 2023-09-22T13:38:08.256 INFO:tasks.workunit.client.0.smithi069.stderr:++ '[' -z '' ']' 2023-09-22T13:38:08.257 INFO:tasks.workunit.client.0.smithi069.stderr:++ _OLD_VIRTUAL_PS1= 2023-09-22T13:38:08.257 INFO:tasks.workunit.client.0.smithi069.stderr:++ '[' 'x(rgw) ' '!=' x ']' 2023-09-22T13:38:08.257 INFO:tasks.workunit.client.0.smithi069.stderr:++ PS1='(rgw) ' 2023-09-22T13:38:08.257 INFO:tasks.workunit.client.0.smithi069.stderr:++ export PS1 2023-09-22T13:38:08.257 INFO:tasks.workunit.client.0.smithi069.stderr:++ '[' -n /usr/bin/bash -o -n '' ']' 2023-09-22T13:38:08.257 INFO:tasks.workunit.client.0.smithi069.stderr:++ hash -r 2023-09-22T13:38:08.257 INFO:tasks.workunit.client.0.smithi069.stderr:+ pip install pip --upgrade 2023-09-22T13:38:08.763 INFO:tasks.workunit.client.0.smithi069.stdout:Collecting pip 2023-09-22T13:38:08.929 INFO:tasks.workunit.client.0.smithi069.stdout: Downloading https://files.pythonhosted.org/packages/a4/6d/6463d49a933f547439d6b5b98b46af8742cc03ae83543e4d7688c2420f8b/pip-21.3.1-py3-none-any.whl (1.7MB) 2023-09-22T13:38:09.213 INFO:tasks.workunit.client.0.smithi069.stdout:Installing collected packages: pip 2023-09-22T13:38:09.213 INFO:tasks.workunit.client.0.smithi069.stdout: Found existing installation: pip 9.0.3 2023-09-22T13:38:09.361 INFO:tasks.workunit.client.0.smithi069.stdout: Uninstalling pip-9.0.3: 2023-09-22T13:38:09.397 INFO:tasks.workunit.client.0.smithi069.stdout: Successfully uninstalled pip-9.0.3 2023-09-22T13:38:10.037 INFO:tasks.workunit.client.0.smithi069.stdout:Successfully installed pip-21.3.1 2023-09-22T13:38:10.345 INFO:tasks.workunit.client.0.smithi069.stderr:You are using pip version 21.3.1, however version 23.2.1 is available. 2023-09-22T13:38:10.345 INFO:tasks.workunit.client.0.smithi069.stderr:You should consider upgrading via the 'pip install --upgrade pip' command. 2023-09-22T13:38:10.376 INFO:tasks.workunit.client.0.smithi069.stderr:+ pip install boto3 2023-09-22T13:38:11.705 INFO:tasks.workunit.client.0.smithi069.stdout:Collecting boto3 2023-09-22T13:38:11.761 INFO:tasks.workunit.client.0.smithi069.stdout: Downloading boto3-1.23.10-py3-none-any.whl (132 kB) 2023-09-22T13:38:11.887 INFO:tasks.workunit.client.0.smithi069.stdout:Collecting jmespath<2.0.0,>=0.7.1 2023-09-22T13:38:11.903 INFO:tasks.workunit.client.0.smithi069.stdout: Downloading jmespath-0.10.0-py2.py3-none-any.whl (24 kB) 2023-09-22T13:38:11.987 INFO:tasks.workunit.client.0.smithi069.stdout:Collecting s3transfer<0.6.0,>=0.5.0 2023-09-22T13:38:12.004 INFO:tasks.workunit.client.0.smithi069.stdout: Downloading s3transfer-0.5.2-py3-none-any.whl (79 kB) 2023-09-22T13:38:13.009 INFO:tasks.workunit.client.0.smithi069.stdout:Collecting botocore<1.27.0,>=1.26.10 2023-09-22T13:38:13.037 INFO:tasks.workunit.client.0.smithi069.stdout: Downloading botocore-1.26.10-py3-none-any.whl (8.8 MB) 2023-09-22T13:38:14.254 INFO:tasks.workunit.client.0.smithi069.stdout:Collecting urllib3<1.27,>=1.25.4 2023-09-22T13:38:14.280 INFO:tasks.workunit.client.0.smithi069.stdout: Downloading urllib3-1.26.16-py2.py3-none-any.whl (143 kB) 2023-09-22T13:38:14.394 INFO:tasks.workunit.client.0.smithi069.stdout:Collecting python-dateutil<3.0.0,>=2.1 2023-09-22T13:38:14.424 INFO:tasks.workunit.client.0.smithi069.stdout: Downloading python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB) 2023-09-22T13:38:14.678 INFO:tasks.workunit.client.0.smithi069.stdout:Collecting six>=1.5 2023-09-22T13:38:14.707 INFO:tasks.workunit.client.0.smithi069.stdout: Downloading six-1.16.0-py2.py3-none-any.whl (11 kB) 2023-09-22T13:38:14.791 INFO:tasks.workunit.client.0.smithi069.stdout:Installing collected packages: six, urllib3, python-dateutil, jmespath, botocore, s3transfer, boto3 2023-09-22T13:38:15.802 INFO:tasks.workunit.client.0.smithi069.stdout:Successfully installed boto3-1.23.10 botocore-1.26.10 jmespath-0.10.0 python-dateutil-2.8.2 s3transfer-0.5.2 six-1.16.0 urllib3-1.26.16 2023-09-22T13:38:16.104 INFO:tasks.workunit.client.0.smithi069.stderr:+ /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/bin/python3 /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_bucket_check.py 2023-09-22T13:38:16.238 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin user create --uid check-tester --display-name "Check Testing" --access-key OJODXSLNX4LUNHQG99PA --secret 3l6ffld34qaymfomuh832j94738aie2x4p2o8h6n 2023-09-22T13:38:16.267 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:16.267 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:16.275 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.274+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.277 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.276+0000 7f9a61603500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.277 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.276+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.277 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.276+0000 7f9a61603500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.277 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.276+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.277 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.277+0000 7f9a61603500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.278 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.277+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.278 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.278+0000 7f9a61603500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:16.290 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.289+0000 7f9a61603500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:16.290 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.289+0000 7f9a61603500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:16.290 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.289+0000 7f9a61603500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:16.291 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.289+0000 7f9a61603500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:16.291 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.289+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.291 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.290+0000 7f9a61603500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:16.291 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.290+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.293 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.290+0000 7f9a61603500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:16.293 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.290+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.293 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.290+0000 7f9a61603500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:16.293 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.290+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.293 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.291+0000 7f9a61603500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.291+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.291+0000 7f9a61603500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.291+0000 7f9a61603500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.291+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.291+0000 7f9a61603500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.291+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.291+0000 7f9a61603500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.291+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.292+0000 7f9a61603500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.292+0000 7f9a61603500 20 zonegroup default 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.292+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.294 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.292+0000 7f9a61603500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.295 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.292+0000 7f9a61603500 10 Cannot find current period zone using local zone 2023-09-22T13:38:16.295 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.292+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.295 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.292+0000 7f9a61603500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.295 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.292+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.295 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.292+0000 7f9a61603500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:16.295 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.292+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.295 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.293+0000 7f9a61603500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:16.295 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.293+0000 7f9a61603500 20 zone default found 2023-09-22T13:38:16.295 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.293+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.296 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.293+0000 7f9a61603500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.296 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.293+0000 7f9a61603500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.296 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.293+0000 7f9a61603500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.296 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.293+0000 7f9a61603500 20 started sync module instance, tier type = 2023-09-22T13:38:16.296 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.293+0000 7f9a61603500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:16.306 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.306+0000 7f9a61603500 20 add_watcher() i=0 2023-09-22T13:38:16.306 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.306+0000 7f9a61603500 20 add_watcher() i=1 2023-09-22T13:38:16.307 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.306+0000 7f9a61603500 20 add_watcher() i=2 2023-09-22T13:38:16.307 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.306+0000 7f9a61603500 20 add_watcher() i=3 2023-09-22T13:38:16.307 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.306+0000 7f9a61603500 20 add_watcher() i=4 2023-09-22T13:38:16.307 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.306+0000 7f9a61603500 20 add_watcher() i=5 2023-09-22T13:38:16.307 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.306+0000 7f9a61603500 20 add_watcher() i=6 2023-09-22T13:38:16.310 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.309+0000 7f9a61603500 20 add_watcher() i=7 2023-09-22T13:38:16.310 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.309+0000 7f9a61603500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:16.310 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.310+0000 7f9a4c2e4700 0 -- 172.21.15.69:0/3254119355 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x55cce28da7e0 msgr2=0x55cce28dcbd0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:16.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.311+0000 7f9a1e7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:16.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.312+0000 7f9a61603500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:16.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.312+0000 7f9a61603500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:16.316 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.316+0000 7f9a4cae5700 0 -- 172.21.15.69:0/2404884421 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x55cce2919b50 msgr2=0x55cce290e870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:16.318 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.318+0000 7f9a61603500 5 note: GC not initialized 2023-09-22T13:38:16.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f998d7fa700 20 reqs_thread_entry: start 2023-09-22T13:38:16.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f9986ffd700 20 reqs_thread_entry: start 2023-09-22T13:38:16.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f9a61603500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:16.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f9a61603500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:16.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f99857fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:46 2023 2023-09-22T13:38:16.321 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:16.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f9984ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:16.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f9a61603500 20 get_system_obj_state: rctx=0x7ffcce4bb4a8 obj=default.rgw.meta:users.uid:check-tester state=0x55cce2922050 s->prefetch_data=0 2023-09-22T13:38:16.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f9a61603500 10 cache get: name=default.rgw.meta+users.uid+check-tester : miss 2023-09-22T13:38:16.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f9a61603500 10 cache put: name=default.rgw.meta+users.uid+check-tester info.flags=0x0 2023-09-22T13:38:16.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f9a61603500 10 adding default.rgw.meta+users.uid+check-tester to cache LRU end 2023-09-22T13:38:16.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f9a61603500 20 get_system_obj_state: rctx=0x7ffcce4bb4b8 obj=default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA state=0x55cce2922050 s->prefetch_data=0 2023-09-22T13:38:16.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.319+0000 7f9a61603500 10 cache get: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA : miss 2023-09-22T13:38:16.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.320+0000 7f9a61603500 10 cache put: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA info.flags=0x0 2023-09-22T13:38:16.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.320+0000 7f9a61603500 10 adding default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA to cache LRU end 2023-09-22T13:38:16.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.320+0000 7f9a61603500 20 get_system_obj_state: rctx=0x7ffcce4badb8 obj=default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA state=0x55cce2922050 s->prefetch_data=0 2023-09-22T13:38:16.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.320+0000 7f9a61603500 10 cache get: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA : hit (negative entry) 2023-09-22T13:38:16.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.320+0000 7f9a61603500 20 get_system_obj_state: rctx=0x7ffcce4bb0f8 obj=default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA state=0x55cce2922050 s->prefetch_data=0 2023-09-22T13:38:16.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.320+0000 7f9a61603500 10 cache get: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA : hit (negative entry) 2023-09-22T13:38:16.323 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.321+0000 7f9a61603500 10 cache put: name=default.rgw.meta+users.uid+check-tester info.flags=0x17 2023-09-22T13:38:16.323 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.321+0000 7f9a61603500 10 moving default.rgw.meta+users.uid+check-tester to cache LRU end 2023-09-22T13:38:16.323 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.321+0000 7f9a61603500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.uid:check-tester, ofs0, ns] 2023-09-22T13:38:16.323 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.322+0000 7f9a4aae1700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 163208757248 cookie 94338462694960 notifier 4701 bl.length()=559 2023-09-22T13:38:16.323 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.322+0000 7f9a4aae1700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+check-tester info.flags=0x17 2023-09-22T13:38:16.323 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.322+0000 7f9a4aae1700 10 rgw watcher librados: moving default.rgw.meta+users.uid+check-tester to cache LRU end 2023-09-22T13:38:16.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.324+0000 7f9a61603500 10 cache put: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA info.flags=0x7 2023-09-22T13:38:16.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.324+0000 7f9a61603500 10 moving default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA to cache LRU end 2023-09-22T13:38:16.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.324+0000 7f9a61603500 10 distributing notification oid=default.rgw.control:notify.0 cni=[op: 0, obj: default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA, ofs0, ns] 2023-09-22T13:38:16.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.325+0000 7f9a4aae1700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 163208757248 cookie 94338462685888 notifier 4701 bl.length()=186 2023-09-22T13:38:16.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.325+0000 7f9a4aae1700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA info.flags=0x7 2023-09-22T13:38:16.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.325+0000 7f9a4aae1700 10 rgw watcher librados: moving default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA to cache LRU end 2023-09-22T13:38:16.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.327+0000 7f9a61603500 20 remove_watcher() i=0 2023-09-22T13:38:16.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.327+0000 7f9a61603500 2 removed watcher, disabling cache 2023-09-22T13:38:16.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.328+0000 7f9a61603500 20 remove_watcher() i=1 2023-09-22T13:38:16.332 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.331+0000 7f9a61603500 20 remove_watcher() i=2 2023-09-22T13:38:16.333 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.332+0000 7f9a61603500 20 remove_watcher() i=3 2023-09-22T13:38:16.334 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.333+0000 7f9a61603500 20 remove_watcher() i=4 2023-09-22T13:38:16.335 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.334+0000 7f9a61603500 20 remove_watcher() i=5 2023-09-22T13:38:16.336 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.335+0000 7f9a61603500 20 remove_watcher() i=6 2023-09-22T13:38:16.337 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.336+0000 7f9a61603500 20 remove_watcher() i=7 2023-09-22T13:38:16.338 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.337+0000 7f9a1e7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:16.707 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/ 2023-09-22T13:38:16.707 INFO:tasks.workunit.client.0.smithi069.stderr: warnings.warn(warning, PythonDeprecationWarning) 2023-09-22T13:38:16.708 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.708 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.708 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.708 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.708 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.708 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.708 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.708 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.708 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.708 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.708 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.709 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.710 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.710 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.710 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.710 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.710 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.710 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.710 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:16.710 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:16.710 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: bucket check unlinked does not report normal entries 2023-09-22T13:38:16.710 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:16.711 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --min-age-hours 0 --dump-keys 2023-09-22T13:38:16.734 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:16.734 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:16.742 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.742+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.744 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.744+0000 7f15cd161500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.744 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.744+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.745 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.744+0000 7f15cd161500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.745 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.744+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.745 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.744+0000 7f15cd161500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.745 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.744+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.746 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.745+0000 7f15cd161500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:16.757 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.756+0000 7f15cd161500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:16.757 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.756+0000 7f15cd161500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:16.757 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.756+0000 7f15cd161500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:16.757 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.756+0000 7f15cd161500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:16.757 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.756+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.758 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.757+0000 7f15cd161500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:16.758 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.757+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.757+0000 7f15cd161500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:16.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.757+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.757+0000 7f15cd161500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:16.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.757+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:16.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:16.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.761 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:16.761 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.761 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:16.761 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 20 zonegroup default 2023-09-22T13:38:16.761 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.758+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.761 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.759+0000 7f15cd161500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.761 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.759+0000 7f15cd161500 10 Cannot find current period zone using local zone 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.759+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.759+0000 7f15cd161500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.759+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.759+0000 7f15cd161500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.759+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.759+0000 7f15cd161500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.759+0000 7f15cd161500 20 zone default found 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.760+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.760+0000 7f15cd161500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.760+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.760+0000 7f15cd161500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:16.762 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.760+0000 7f15cd161500 20 started sync module instance, tier type = 2023-09-22T13:38:16.763 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.760+0000 7f15cd161500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:16.773 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.773+0000 7f15cd161500 20 add_watcher() i=0 2023-09-22T13:38:16.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.773+0000 7f15cd161500 20 add_watcher() i=1 2023-09-22T13:38:16.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.773+0000 7f15cd161500 20 add_watcher() i=2 2023-09-22T13:38:16.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.773+0000 7f15cd161500 20 add_watcher() i=3 2023-09-22T13:38:16.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.773+0000 7f15cd161500 20 add_watcher() i=4 2023-09-22T13:38:16.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.773+0000 7f15cd161500 20 add_watcher() i=5 2023-09-22T13:38:16.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.773+0000 7f15cd161500 20 add_watcher() i=6 2023-09-22T13:38:16.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.774+0000 7f15cd161500 20 add_watcher() i=7 2023-09-22T13:38:16.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.774+0000 7f15cd161500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:16.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.774+0000 7f15b7e42700 0 -- 172.21.15.69:0/1839330745 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x5595e1656c00 msgr2=0x5595e1658ff0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:16.776 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.776+0000 7f1589fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:16.777 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.776+0000 7f15cd161500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:16.777 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.776+0000 7f15cd161500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:16.781 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.780+0000 7f15b7e42700 0 -- 172.21.15.69:0/3824079831 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x5595e16a4170 msgr2=0x5595e1693800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:16.783 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.783+0000 7f15cd161500 5 note: GC not initialized 2023-09-22T13:38:16.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.784+0000 7f1500ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:16.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.784+0000 7f14f67fc700 20 reqs_thread_entry: start 2023-09-22T13:38:16.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.784+0000 7f15cd161500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:16.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.784+0000 7f15cd161500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:16.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.784+0000 7f14f4ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:46 2023 2023-09-22T13:38:16.785 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:16.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.784+0000 7f14dbfff700 20 reqs_thread_entry: start 2023-09-22T13:38:16.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.784+0000 7f15cd161500 20 get_system_obj_state: rctx=0x7ffda0a22a78 obj=default.rgw.meta:root:check-bucket state=0x5595e169a7c0 s->prefetch_data=0 2023-09-22T13:38:16.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.784+0000 7f15cd161500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:38:16.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.785+0000 7f15cd161500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:38:16.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.785+0000 7f15cd161500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:16.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.785+0000 7f15cd161500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:16.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.785+0000 7f15cd161500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:16.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.785+0000 7f15cd161500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:38:16.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.785+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.785+0000 7f15cd161500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:16.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.785+0000 7f15cd161500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:38:16.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.785+0000 7f15cd161500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:16.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.785+0000 7f15cd161500 20 get_system_obj_state: rctx=0x7ffda0a22b48 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x5595e169a7c0 s->prefetch_data=0 2023-09-22T13:38:16.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.785+0000 7f15cd161500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : miss 2023-09-22T13:38:16.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x16 2023-09-22T13:38:16.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:16.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:16.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:16.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:16.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:16.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:16.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:38:16.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 20 rados->read ofs=0 len=0 2023-09-22T13:38:16.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:16.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x13 2023-09-22T13:38:16.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:16.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:16.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:16.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.786+0000 7f15cd161500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 2023-09-22T13:38:16.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.787+0000 7f15cd161500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:16.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.787+0000 7f15cd161500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:38:16.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.787+0000 7f15cd161500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:16.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.787+0000 7f15cd161500 1 NOTICE: finished shard 1 (0 entries found) 2023-09-22T13:38:16.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.787+0000 7f15cd161500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:16.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.788+0000 7f15cd161500 1 NOTICE: finished shard 2 (0 entries found) 2023-09-22T13:38:16.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.788+0000 7f15cd161500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:16.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.788+0000 7f15cd161500 1 NOTICE: finished shard 3 (0 entries found) 2023-09-22T13:38:16.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.788+0000 7f15cd161500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:16.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.789+0000 7f15cd161500 1 NOTICE: finished shard 4 (0 entries found) 2023-09-22T13:38:16.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.789+0000 7f15cd161500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:16.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.789+0000 7f15cd161500 1 NOTICE: finished shard 5 (0 entries found) 2023-09-22T13:38:16.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.789+0000 7f15cd161500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:16.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.790+0000 7f15cd161500 1 NOTICE: finished shard 6 (0 entries found) 2023-09-22T13:38:16.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.790+0000 7f15cd161500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:16.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.790+0000 7f15cd161500 1 NOTICE: finished shard 7 (0 entries found) 2023-09-22T13:38:16.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.790+0000 7f15cd161500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:16.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.791+0000 7f15cd161500 1 NOTICE: finished shard 8 (0 entries found) 2023-09-22T13:38:16.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.791+0000 7f15cd161500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:16.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.791+0000 7f15cd161500 1 NOTICE: finished shard 9 (0 entries found) 2023-09-22T13:38:16.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.791+0000 7f15cd161500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:16.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.792+0000 7f15cd161500 1 NOTICE: finished shard 10 (0 entries found) 2023-09-22T13:38:16.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.792+0000 7f15cd161500 1 NOTICE: finished all shards (0 entries found) 2023-09-22T13:38:16.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.793+0000 7f15cd161500 20 remove_watcher() i=0 2023-09-22T13:38:16.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.793+0000 7f15cd161500 2 removed watcher, disabling cache 2023-09-22T13:38:16.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.794+0000 7f15cd161500 20 remove_watcher() i=1 2023-09-22T13:38:16.796 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.795+0000 7f15cd161500 20 remove_watcher() i=2 2023-09-22T13:38:16.797 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.797+0000 7f15cd161500 20 remove_watcher() i=3 2023-09-22T13:38:16.798 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.798+0000 7f15cd161500 20 remove_watcher() i=4 2023-09-22T13:38:16.799 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.799+0000 7f15cd161500 20 remove_watcher() i=5 2023-09-22T13:38:16.800 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.800+0000 7f15cd161500 20 remove_watcher() i=6 2023-09-22T13:38:16.801 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.801+0000 7f15cd161500 20 remove_watcher() i=7 2023-09-22T13:38:16.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:16.801+0000 7f1589fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:16.808 INFO:tasks.workunit.client.0.smithi069.stderr:ceph config set client rgw_debug_inject_set_olh_err 2 2023-09-22T13:38:17.197 INFO:tasks.workunit.client.0.smithi069.stderr:ceph config set client rgw_debug_inject_olh_cancel_modification_err true 2023-09-22T13:38:18.750 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:18.750 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:18.750 INFO:tasks.workunit.client.0.smithi069.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:38:18.751 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:38:18.776 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:18.777 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:18.784 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.784+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.786 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.785+0000 7f5ba326f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.786 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.785+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.786+0000 7f5ba326f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.786+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.786+0000 7f5ba326f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.786+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.787+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:18.799 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.798+0000 7f5ba326f500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:18.799 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.798+0000 7f5ba326f500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:18.799 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.798+0000 7f5ba326f500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:18.799 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.798+0000 7f5ba326f500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:18.799 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.798+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.799 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.798+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:18.799 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.798+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.800 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.799+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:18.800 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.799+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.800 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.799+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:18.800 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.799+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.801 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.799+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:18.801 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.799+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.801 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.800+0000 7f5ba326f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.801 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.800+0000 7f5ba326f500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:18.801 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.800+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.801 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.800+0000 7f5ba326f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.801 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.800+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.800+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:18.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.800+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.801+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.801+0000 7f5ba326f500 20 zonegroup default 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.801+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.801+0000 7f5ba326f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.801+0000 7f5ba326f500 10 Cannot find current period zone using local zone 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.801+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.801+0000 7f5ba326f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.801+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.801+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.801+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.802+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.802+0000 7f5ba326f500 20 zone default found 2023-09-22T13:38:18.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.802+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.804 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.802+0000 7f5ba326f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.804 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.802+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.804 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.802+0000 7f5ba326f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.804 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.802+0000 7f5ba326f500 20 started sync module instance, tier type = 2023-09-22T13:38:18.804 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.802+0000 7f5ba326f500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:18.817 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.816+0000 7f5ba326f500 20 add_watcher() i=0 2023-09-22T13:38:18.817 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.816+0000 7f5ba326f500 20 add_watcher() i=1 2023-09-22T13:38:18.817 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.816+0000 7f5ba326f500 20 add_watcher() i=2 2023-09-22T13:38:18.817 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.816+0000 7f5ba326f500 20 add_watcher() i=3 2023-09-22T13:38:18.817 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.816+0000 7f5ba326f500 20 add_watcher() i=4 2023-09-22T13:38:18.817 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.816+0000 7f5ba326f500 20 add_watcher() i=5 2023-09-22T13:38:18.817 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.816+0000 7f5ba326f500 20 add_watcher() i=6 2023-09-22T13:38:18.818 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.818+0000 7f5b8e751700 0 -- 172.21.15.69:0/333106750 >> [v2:172.21.15.69:6800/145165,v1:172.21.15.69:6801/145165] conn(0x7f5b7801f030 msgr2=0x7f5b7802b940 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:18.819 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.818+0000 7f5ba326f500 20 add_watcher() i=7 2023-09-22T13:38:18.819 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.818+0000 7f5ba326f500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:18.821 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.820+0000 7f5b5ffdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:18.821 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.820+0000 7f5ba326f500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:18.821 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.820+0000 7f5ba326f500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:18.825 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.824+0000 7f5b8df50700 0 -- 172.21.15.69:0/3152523876 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x55aa809602c0 msgr2=0x55aa8096f520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:18.826 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.826+0000 7f5ba326f500 5 note: GC not initialized 2023-09-22T13:38:18.828 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.827+0000 7f5aceffd700 20 reqs_thread_entry: start 2023-09-22T13:38:18.828 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.827+0000 7f5accff9700 20 reqs_thread_entry: start 2023-09-22T13:38:18.828 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.827+0000 7f5ba326f500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:18.828 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.827+0000 7f5ba326f500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:18.828 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.827+0000 7f5ac2ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:49 2023 2023-09-22T13:38:18.829 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:18.829 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.827+0000 7f5ac27fc700 20 reqs_thread_entry: start 2023-09-22T13:38:18.829 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.827+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0cc0 obj=default.rgw.meta:root:check-bucket state=0x55aa80934470 s->prefetch_data=0 2023-09-22T13:38:18.829 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.828+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.829 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.828+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.830 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.828+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:18.830 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.828+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0cc0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.830 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.829+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.830 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.829+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.830 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.829+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.830 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.829+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.830 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.829+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.831 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.829+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.831 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.829+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0a90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.831 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.829+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.831 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.829+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.831 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.829+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.831 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.830+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.832 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.830+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.832 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.830+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.832 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.830+0000 7f5ba326f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:18.832 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.831+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0a90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.832 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.831+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.832 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.831+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.832 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.831+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.832 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.831+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.833 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.832+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.833 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.832+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.833 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.832+0000 7f5ba326f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:18.833 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.832+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0a90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.833 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.833+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.834 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.833+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.834 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.833+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.834 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.833+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.835 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.833+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.835 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.833+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.835 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.833+0000 7f5ba326f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:18.835 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.833+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0a90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.835 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.834+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.835 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.834+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.835 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.834+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.835 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.834+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.835 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.834+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.836 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.834+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.836 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.834+0000 7f5ba326f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:18.836 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.835+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0a90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.836 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.835+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.837 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.835+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.837 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.835+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.837 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.835+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.837 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.836+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.837 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.836+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.837 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.836+0000 7f5ba326f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:18.837 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.836+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0a90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.838 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.837+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.838 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.837+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.838 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.837+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.838 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.837+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.838 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.837+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.838 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.837+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.838 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.837+0000 7f5ba326f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:18.839 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.838+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0a90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.839 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.838+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.839 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.838+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.839 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.838+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.839 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.838+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.840 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.839+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.840 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.839+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.840 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.839+0000 7f5ba326f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:18.840 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.839+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0a90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.840 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.840+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.841 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.840+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.841 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.840+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.841 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.840+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.841 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.840+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.841 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.840+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.841 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.840+0000 7f5ba326f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:18.842 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.840+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0a90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.842 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.841+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.842 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.841+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.842 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.841+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.842 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.841+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.842 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.841+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.842 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.841+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.842 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.841+0000 7f5ba326f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:18.843 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.842+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0a90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.843 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.843+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.843 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.843+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.844 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.843+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.844 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.843+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.844 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.843+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.844 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.843+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.844 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.843+0000 7f5ba326f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:18.845 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.844+0000 7f5ba326f500 20 get_system_obj_state: rctx=0x7ffecb8a0a90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55aa808240c0 s->prefetch_data=0 2023-09-22T13:38:18.845 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.844+0000 7f5ba326f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:18.845 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.844+0000 7f5ba326f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:18.845 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.844+0000 7f5ba326f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:18.846 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.844+0000 7f5ba326f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.846 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.845+0000 7f5ba326f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:18.846 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.845+0000 7f5ba326f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:18.846 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.845+0000 7f5ba326f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:18.847 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.847+0000 7f5ba326f500 20 remove_watcher() i=0 2023-09-22T13:38:18.847 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.847+0000 7f5ba326f500 2 removed watcher, disabling cache 2023-09-22T13:38:18.848 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.848+0000 7f5ba326f500 20 remove_watcher() i=1 2023-09-22T13:38:18.849 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.849+0000 7f5ba326f500 20 remove_watcher() i=2 2023-09-22T13:38:18.850 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.850+0000 7f5ba326f500 20 remove_watcher() i=3 2023-09-22T13:38:18.851 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.851+0000 7f5ba326f500 20 remove_watcher() i=4 2023-09-22T13:38:18.852 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.852+0000 7f5ba326f500 20 remove_watcher() i=5 2023-09-22T13:38:18.853 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.853+0000 7f5ba326f500 20 remove_watcher() i=6 2023-09-22T13:38:18.854 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.854+0000 7f5ba326f500 20 remove_watcher() i=7 2023-09-22T13:38:18.855 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.854+0000 7f5b5ffdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:18.919 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:18.919 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:18.919 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:18.919 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:18.919 INFO:tasks.workunit.client.0.smithi069.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:38:18.920 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:38:18.943 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:18.944 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:18.951 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.950+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.953 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.951+0000 7fc48c08c500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.953 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.951+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.953 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.952+0000 7fc48c08c500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.954 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.952+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.954 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.952+0000 7fc48c08c500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.954 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.952+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.954 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.953+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:18.965 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.963+0000 7fc48c08c500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:18.965 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.963+0000 7fc48c08c500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:18.965 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.963+0000 7fc48c08c500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:18.965 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.963+0000 7fc48c08c500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:18.965 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.963+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.965 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.963+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:18.965 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.963+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.966 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.964+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:18.967 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.964+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.967 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.964+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:18.967 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.964+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.967 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.965+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:18.967 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.965+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.967 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.965+0000 7fc48c08c500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.967 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.965+0000 7fc48c08c500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:18.967 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.965+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.967 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.965+0000 7fc48c08c500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.967 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.965+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.968 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.966+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:18.968 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.966+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.969 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.966+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:18.969 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.966+0000 7fc48c08c500 20 zonegroup default 2023-09-22T13:38:18.969 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.966+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.969 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.966+0000 7fc48c08c500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.969 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.966+0000 7fc48c08c500 10 Cannot find current period zone using local zone 2023-09-22T13:38:18.969 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.966+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.969 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.966+0000 7fc48c08c500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.969 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.966+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.969 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.967+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:18.969 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.967+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.970 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.967+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:18.970 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.967+0000 7fc48c08c500 20 zone default found 2023-09-22T13:38:18.970 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.967+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.970 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.968+0000 7fc48c08c500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.971 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.968+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:18.971 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.968+0000 7fc48c08c500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:18.971 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.968+0000 7fc48c08c500 20 started sync module instance, tier type = 2023-09-22T13:38:18.971 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.968+0000 7fc48c08c500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:18.987 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.986+0000 7fc48c08c500 20 add_watcher() i=0 2023-09-22T13:38:18.988 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.986+0000 7fc48c08c500 20 add_watcher() i=1 2023-09-22T13:38:18.988 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.986+0000 7fc48c08c500 20 add_watcher() i=2 2023-09-22T13:38:18.988 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.986+0000 7fc48c08c500 20 add_watcher() i=3 2023-09-22T13:38:18.988 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.986+0000 7fc48c08c500 20 add_watcher() i=4 2023-09-22T13:38:18.988 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.986+0000 7fc48c08c500 20 add_watcher() i=5 2023-09-22T13:38:18.988 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.986+0000 7fc48c08c500 20 add_watcher() i=6 2023-09-22T13:38:18.990 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.988+0000 7fc48c08c500 20 add_watcher() i=7 2023-09-22T13:38:18.990 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.988+0000 7fc48c08c500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:18.990 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.988+0000 7fc477d6f700 0 -- 172.21.15.69:0/141168921 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x55b9ea8fde70 msgr2=0x55b9ea900260 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:38:18.992 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.990+0000 7fc3c3fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:18.992 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.991+0000 7fc48c08c500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:18.992 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.991+0000 7fc48c08c500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:18.996 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.995+0000 7fc477d6f700 0 -- 172.21.15.69:0/3858220484 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x55b9ea91c0d0 msgr2=0x55b9ea949e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:18.999 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.997+0000 7fc48c08c500 5 note: GC not initialized 2023-09-22T13:38:19.000 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.998+0000 7fc3b2ffd700 20 reqs_thread_entry: start 2023-09-22T13:38:19.000 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.998+0000 7fc3b0ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:19.001 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.998+0000 7fc48c08c500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:19.001 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.999+0000 7fc48c08c500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:19.001 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.999+0000 7fc3a2ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:49 2023 2023-09-22T13:38:19.001 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:19.001 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.999+0000 7fc3a27fc700 20 reqs_thread_entry: start 2023-09-22T13:38:19.001 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.999+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9e30 obj=default.rgw.meta:root:check-bucket state=0x55b9ea8b4be0 s->prefetch_data=0 2023-09-22T13:38:19.001 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.999+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.001 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.999+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.002 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.999+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:19.002 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:18.999+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9e30 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.002 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.000+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.002 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.000+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.002 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.000+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.002 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.000+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.003 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.001+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.003 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.001+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.003 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.001+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9c00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.003 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.001+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.004 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.001+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.004 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.001+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.004 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.001+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.004 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.002+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.004 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.002+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.004 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.002+0000 7fc48c08c500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:19.005 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.002+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9c00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.005 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.003+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.005 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.003+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.005 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.003+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.005 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.003+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.005 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.003+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.006 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.003+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.006 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.003+0000 7fc48c08c500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:19.006 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.004+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9c00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.006 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.004+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.006 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.004+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.007 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.004+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.007 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.004+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.007 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.005+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.007 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.005+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.007 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.005+0000 7fc48c08c500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:19.008 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.005+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9c00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.008 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.006+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.008 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.006+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.008 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.006+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.008 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.006+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.008 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.006+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.008 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.006+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.008 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.006+0000 7fc48c08c500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:19.009 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.007+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9c00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.009 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.007+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.009 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.007+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.009 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.007+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.009 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.007+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.009 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.007+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.009 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.007+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.009 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.007+0000 7fc48c08c500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:19.010 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.008+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9c00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.010 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.008+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.010 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.008+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.011 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.008+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.011 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.008+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.011 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.009+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.011 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.009+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.011 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.009+0000 7fc48c08c500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:19.011 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.009+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9c00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.011 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.010+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.012 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.010+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.012 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.010+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.012 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.010+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.012 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.010+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.012 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.010+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.012 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.010+0000 7fc48c08c500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:19.013 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.011+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9c00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.013 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.011+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.013 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.011+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.013 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.011+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.013 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.011+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.014 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.012+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.014 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.012+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.014 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.012+0000 7fc48c08c500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:19.014 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.012+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9c00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.014 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.013+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.015 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.013+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.015 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.013+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.015 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.013+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.015 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.013+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.013+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.014+0000 7fc48c08c500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:19.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.014+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9c00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.014+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.014+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.014+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.014+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.017 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.015+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.017 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.015+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.017 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.015+0000 7fc48c08c500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:19.017 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.016+0000 7fc48c08c500 20 get_system_obj_state: rctx=0x7ffe36dc9c00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b9ea8c4fc0 s->prefetch_data=0 2023-09-22T13:38:19.018 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.016+0000 7fc48c08c500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.018 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.016+0000 7fc48c08c500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.018 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.016+0000 7fc48c08c500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.018 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.016+0000 7fc48c08c500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.019 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.017+0000 7fc48c08c500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.019 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.017+0000 7fc48c08c500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.019 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.017+0000 7fc48c08c500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:19.020 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.018+0000 7fc48c08c500 20 remove_watcher() i=0 2023-09-22T13:38:19.020 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.018+0000 7fc48c08c500 2 removed watcher, disabling cache 2023-09-22T13:38:19.021 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.020+0000 7fc48c08c500 20 remove_watcher() i=1 2023-09-22T13:38:19.022 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.021+0000 7fc48c08c500 20 remove_watcher() i=2 2023-09-22T13:38:19.023 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.022+0000 7fc48c08c500 20 remove_watcher() i=3 2023-09-22T13:38:19.024 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.023+0000 7fc48c08c500 20 remove_watcher() i=4 2023-09-22T13:38:19.025 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.024+0000 7fc48c08c500 20 remove_watcher() i=5 2023-09-22T13:38:19.026 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.025+0000 7fc48c08c500 20 remove_watcher() i=6 2023-09-22T13:38:19.027 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.026+0000 7fc48c08c500 20 remove_watcher() i=7 2023-09-22T13:38:19.028 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.027+0000 7fc3c3fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:19.090 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:19.090 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:19.090 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:19.090 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:19.090 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:19.090 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:19.090 INFO:tasks.workunit.client.0.smithi069.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:38:19.091 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object e 2023-09-22T13:38:19.116 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:19.116 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:19.125 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.123+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.127 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.125+0000 7f211797f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.127 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.125+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.128 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.126+0000 7f211797f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.128 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.126+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.128 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.126+0000 7f211797f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.128 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.126+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.129 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.127+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:19.139 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.137+0000 7f211797f500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:19.139 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.137+0000 7f211797f500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:19.139 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.137+0000 7f211797f500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:19.139 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.137+0000 7f211797f500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:19.139 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.137+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.139 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.137+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:19.139 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.137+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.140 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.138+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:19.140 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.138+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.140 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.138+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:19.140 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.138+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.140 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.138+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:19.140 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.138+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.141 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.138+0000 7f211797f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.141 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.138+0000 7f211797f500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:19.141 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.138+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.141 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.139+0000 7f211797f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.141 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.139+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.142 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.139+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:19.142 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.139+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.142 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.139+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:19.142 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.139+0000 7f211797f500 20 zonegroup default 2023-09-22T13:38:19.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.139+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.139+0000 7f211797f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.139+0000 7f211797f500 10 Cannot find current period zone using local zone 2023-09-22T13:38:19.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.139+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.140+0000 7f211797f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.140+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.140+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:19.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.140+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.140+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:19.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.140+0000 7f211797f500 20 zone default found 2023-09-22T13:38:19.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.140+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.144 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.141+0000 7f211797f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.144 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.141+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.144 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.141+0000 7f211797f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.144 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.141+0000 7f211797f500 20 started sync module instance, tier type = 2023-09-22T13:38:19.144 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.141+0000 7f211797f500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:19.146 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.145+0000 7f2103662700 0 -- 172.21.15.69:0/2494674772 >> [v2:172.21.15.174:6816/145171,v1:172.21.15.174:6817/145171] conn(0x7f20f801d370 msgr2=0x7f20f801d7b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:19.154 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.153+0000 7f211797f500 20 add_watcher() i=0 2023-09-22T13:38:19.154 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.153+0000 7f211797f500 20 add_watcher() i=1 2023-09-22T13:38:19.154 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.153+0000 7f211797f500 20 add_watcher() i=2 2023-09-22T13:38:19.154 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.153+0000 7f211797f500 20 add_watcher() i=3 2023-09-22T13:38:19.155 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.153+0000 7f211797f500 20 add_watcher() i=4 2023-09-22T13:38:19.155 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.153+0000 7f211797f500 20 add_watcher() i=5 2023-09-22T13:38:19.155 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.153+0000 7f211797f500 20 add_watcher() i=6 2023-09-22T13:38:19.155 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.154+0000 7f211797f500 20 add_watcher() i=7 2023-09-22T13:38:19.155 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.154+0000 7f211797f500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:19.157 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.156+0000 7f20d3fe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:19.157 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.156+0000 7f211797f500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:19.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.156+0000 7f211797f500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:19.159 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.157+0000 7f2102e61700 0 -- 172.21.15.69:0/3392259231 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x561d59eceb10 msgr2=0x561d59eca450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2023-09-22T13:38:19.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.358+0000 7f2102e61700 0 --2- 172.21.15.69:0/3392259231 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x561d59eceb10 0x561d59eca450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:38:19.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.362+0000 7f211797f500 5 note: GC not initialized 2023-09-22T13:38:19.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.363+0000 7f2042ffd700 20 reqs_thread_entry: start 2023-09-22T13:38:19.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.364+0000 7f2040ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:19.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.364+0000 7f211797f500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:19.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.364+0000 7f211797f500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:19.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.364+0000 7f2036ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:49 2023 2023-09-22T13:38:19.366 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:19.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.364+0000 7f20367fc700 20 reqs_thread_entry: start 2023-09-22T13:38:19.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.364+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533f50 obj=default.rgw.meta:root:check-bucket state=0x561d59d9d0c0 s->prefetch_data=0 2023-09-22T13:38:19.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.364+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.364+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.364+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:19.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.364+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533f50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.365+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.365+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.365+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.365+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.365+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.365+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.365+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533d20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.366+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.366+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.366+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.369 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.366+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.366+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.366+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.366+0000 7f211797f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:19.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.367+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533d20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.367+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.367+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.367+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.367+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.368+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.368+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.368+0000 7f211797f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:19.371 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.368+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533d20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.371 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.369+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.371 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.369+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.371 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.369+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.372 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.369+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.372 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.369+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.372 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.369+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.372 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.369+0000 7f211797f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:19.372 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.370+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533d20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.373 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.370+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.373 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.370+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.373 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.370+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.373 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.370+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.373 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.370+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.373 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.370+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.373 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.370+0000 7f211797f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:19.373 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.371+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533d20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.374 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.371+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.374 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.371+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.374 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.371+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.374 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.371+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.375 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.372+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.375 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.372+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.375 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.372+0000 7f211797f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:19.375 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.372+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533d20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.375 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.372+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.375 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.372+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.375 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.372+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.375 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.372+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.376 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.373+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.376 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.373+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.376 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.373+0000 7f211797f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:19.376 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.374+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533d20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.376 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.374+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.377 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.374+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.377 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.374+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.377 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.374+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.377 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.375+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.377 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.375+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.377 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.375+0000 7f211797f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:19.378 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.375+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533d20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.378 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.376+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.378 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.376+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.378 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.376+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.378 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.376+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.378 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.376+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.378 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.376+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.378 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.376+0000 7f211797f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:19.379 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.377+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533d20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.379 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.377+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.379 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.377+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.380 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.377+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.380 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.377+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.380 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.378+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.380 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.378+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.380 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.378+0000 7f211797f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:19.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.378+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533d20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.378+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.378+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.378+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.378+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.379+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.379+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.379+0000 7f211797f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:19.382 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.380+0000 7f211797f500 20 get_system_obj_state: rctx=0x7ffd6d533d20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x561d59d1f810 s->prefetch_data=0 2023-09-22T13:38:19.382 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.380+0000 7f211797f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.382 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.380+0000 7f211797f500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.382 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.380+0000 7f211797f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.382 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.380+0000 7f211797f500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.382 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.380+0000 7f211797f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.383 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.380+0000 7f211797f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.383 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.380+0000 7f211797f500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:19.384 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.382+0000 7f211797f500 20 remove_watcher() i=0 2023-09-22T13:38:19.384 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.382+0000 7f211797f500 2 removed watcher, disabling cache 2023-09-22T13:38:19.385 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.383+0000 7f211797f500 20 remove_watcher() i=1 2023-09-22T13:38:19.386 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.384+0000 7f211797f500 20 remove_watcher() i=2 2023-09-22T13:38:19.387 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.386+0000 7f211797f500 20 remove_watcher() i=3 2023-09-22T13:38:19.388 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.387+0000 7f211797f500 20 remove_watcher() i=4 2023-09-22T13:38:19.389 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.388+0000 7f211797f500 20 remove_watcher() i=5 2023-09-22T13:38:19.390 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.389+0000 7f211797f500 20 remove_watcher() i=6 2023-09-22T13:38:19.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.390+0000 7f211797f500 20 remove_watcher() i=7 2023-09-22T13:38:19.392 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.390+0000 7f20d3fe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:19.453 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:19.454 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:19.454 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:19.454 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:19.454 INFO:tasks.workunit.client.0.smithi069.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:38:19.455 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object f 2023-09-22T13:38:19.480 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:19.480 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:19.488 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.486+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.490 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.488+0000 7f8afff12500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.490 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.488+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.490 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.488+0000 7f8afff12500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.490 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.488+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.490 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.489+0000 7f8afff12500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.491 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.489+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.491 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.490+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:19.502 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.500+0000 7f8afff12500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:19.502 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.500+0000 7f8afff12500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:19.502 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.500+0000 7f8afff12500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:19.502 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.500+0000 7f8afff12500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:19.502 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.500+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.502 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.500+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:19.502 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.500+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.504 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.501+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:19.504 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.501+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.504 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.501+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:19.504 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.501+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.505 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.501+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:19.505 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.501+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.505 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.505 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:19.505 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.505 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.505 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.505 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:19.505 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.505 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:19.505 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 20 zonegroup default 2023-09-22T13:38:19.506 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.506 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.506 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 10 Cannot find current period zone using local zone 2023-09-22T13:38:19.506 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.502+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.506 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.503+0000 7f8afff12500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.506 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.503+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.507 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.503+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:19.507 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.503+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.507 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.503+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:19.507 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.503+0000 7f8afff12500 20 zone default found 2023-09-22T13:38:19.507 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.503+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.507 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.504+0000 7f8afff12500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.507 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.504+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.508 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.504+0000 7f8afff12500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:19.508 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.504+0000 7f8afff12500 20 started sync module instance, tier type = 2023-09-22T13:38:19.508 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.504+0000 7f8afff12500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:19.525 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.523+0000 7f8afff12500 20 add_watcher() i=0 2023-09-22T13:38:19.525 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.523+0000 7f8afff12500 20 add_watcher() i=1 2023-09-22T13:38:19.525 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.523+0000 7f8afff12500 20 add_watcher() i=2 2023-09-22T13:38:19.525 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.523+0000 7f8afff12500 20 add_watcher() i=3 2023-09-22T13:38:19.525 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.523+0000 7f8afff12500 20 add_watcher() i=4 2023-09-22T13:38:19.526 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.523+0000 7f8afff12500 20 add_watcher() i=5 2023-09-22T13:38:19.526 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.523+0000 7f8afff12500 20 add_watcher() i=6 2023-09-22T13:38:19.527 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.526+0000 7f8afff12500 20 add_watcher() i=7 2023-09-22T13:38:19.527 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.526+0000 7f8afff12500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:19.530 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.529+0000 7f8aeabf3700 0 -- 172.21.15.69:0/162608346 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x55f2c0089d10 msgr2=0x55f2c008c100 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:19.530 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.529+0000 7f8a37fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:19.531 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.529+0000 7f8afff12500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:19.531 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.529+0000 7f8afff12500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:19.535 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.534+0000 7f8aeb3f4700 0 -- 172.21.15.69:0/1486913840 >> [v2:172.21.15.69:3301/0,v1:172.21.15.69:6790/0] conn(0x55f2c00ca360 msgr2=0x55f2c00cdbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:19.537 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.536+0000 7f8afff12500 5 note: GC not initialized 2023-09-22T13:38:19.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.537+0000 7f8a2effd700 20 reqs_thread_entry: start 2023-09-22T13:38:19.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.537+0000 7f8a2cff9700 20 reqs_thread_entry: start 2023-09-22T13:38:19.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.537+0000 7f8afff12500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:19.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.537+0000 7f8afff12500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:19.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.537+0000 7f8a1affd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:49 2023 2023-09-22T13:38:19.540 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:19.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.537+0000 7f8a1a7fc700 20 reqs_thread_entry: start 2023-09-22T13:38:19.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.537+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed13691d0 obj=default.rgw.meta:root:check-bucket state=0x55f2bff9b0c0 s->prefetch_data=0 2023-09-22T13:38:19.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.538+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.538+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.538+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:19.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.538+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed13691d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.541 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.539+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.541 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.539+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.542 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.539+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.542 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.539+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.542 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.539+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.542 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.539+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.542 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.539+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed1368fa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.542 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.539+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.542 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.539+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.542 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.539+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.542 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.539+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.542 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.540+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.543 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.540+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.543 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.540+0000 7f8afff12500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:19.544 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.540+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed1368fa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.544 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.541+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.544 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.541+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.544 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.541+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.544 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.541+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.544 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.541+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.545 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.541+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.545 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.541+0000 7f8afff12500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:19.545 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.542+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed1368fa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.545 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.542+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.545 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.542+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.545 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.542+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.545 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.542+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.545 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.542+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.545 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.542+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.545 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.542+0000 7f8afff12500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:19.546 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.543+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed1368fa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.547 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.543+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.547 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.543+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.547 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.543+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.547 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.543+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.547 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.543+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.547 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.543+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.547 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.543+0000 7f8afff12500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:19.547 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.544+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed1368fa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.547 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.544+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.544+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.544+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.544+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.544+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.544+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.544+0000 7f8afff12500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:19.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.545+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed1368fa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.550 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.545+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.550 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.545+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.550 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.545+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.550 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.545+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.550 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.546+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.550 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.546+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.550 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.546+0000 7f8afff12500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:19.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.546+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed1368fa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.547+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.547+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.547+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.547+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.547+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.547+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.547+0000 7f8afff12500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:19.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.548+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed1368fa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.548+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.548+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.552 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.548+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.552 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.548+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.552 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.548+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.552 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.548+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.552 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.548+0000 7f8afff12500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:19.553 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.549+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed1368fa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.553 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.549+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.554 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.549+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.554 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.549+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.554 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.549+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.554 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.550+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.554 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.550+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.554 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.550+0000 7f8afff12500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:19.554 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.550+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed1368fa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.554 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.551+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.554 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.551+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.554 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.551+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.555 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.551+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.555 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.551+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.555 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.551+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.555 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.551+0000 7f8afff12500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:19.555 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.551+0000 7f8afff12500 20 get_system_obj_state: rctx=0x7ffed1368fa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f2bff1d810 s->prefetch_data=0 2023-09-22T13:38:19.555 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.552+0000 7f8afff12500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:19.555 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.552+0000 7f8afff12500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:19.555 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.552+0000 7f8afff12500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:19.555 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.552+0000 7f8afff12500 20 rados->read ofs=0 len=0 2023-09-22T13:38:19.556 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.552+0000 7f8afff12500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:19.556 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.552+0000 7f8afff12500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:19.556 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.552+0000 7f8afff12500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:19.556 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.554+0000 7f8afff12500 20 remove_watcher() i=0 2023-09-22T13:38:19.556 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.554+0000 7f8afff12500 2 removed watcher, disabling cache 2023-09-22T13:38:19.557 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.555+0000 7f8afff12500 20 remove_watcher() i=1 2023-09-22T13:38:19.558 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.556+0000 7f8afff12500 20 remove_watcher() i=2 2023-09-22T13:38:19.559 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.557+0000 7f8afff12500 20 remove_watcher() i=3 2023-09-22T13:38:19.560 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.559+0000 7f8afff12500 20 remove_watcher() i=4 2023-09-22T13:38:19.561 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.560+0000 7f8afff12500 20 remove_watcher() i=5 2023-09-22T13:38:19.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.561+0000 7f8afff12500 20 remove_watcher() i=6 2023-09-22T13:38:19.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.561+0000 7f8afff12500 20 remove_watcher() i=7 2023-09-22T13:38:19.564 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:19.562+0000 7f8a37fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:19.574 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:19.574 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:19.575 INFO:tasks.workunit.client.0.smithi069.stderr:ceph config rm client rgw_debug_inject_set_olh_err 2023-09-22T13:38:19.964 INFO:tasks.workunit.client.0.smithi069.stderr:ceph config rm client rgw_debug_inject_olh_cancel_modification_err 2023-09-22T13:38:20.354 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: bucket check unlinked finds unlistable entries 2023-09-22T13:38:20.355 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:20.356 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --min-age-hours 0 --dump-keys 2023-09-22T13:38:20.382 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:20.382 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:20.390 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.388+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.390+0000 7fd82ca13500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.390+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.392 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.390+0000 7fd82ca13500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.392 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.390+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.392 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.391+0000 7fd82ca13500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.392 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.391+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.393 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.392+0000 7fd82ca13500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:20.405 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.403+0000 7fd82ca13500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:20.405 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.403+0000 7fd82ca13500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:20.405 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.403+0000 7fd82ca13500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:20.405 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.403+0000 7fd82ca13500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:20.405 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.403+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.406 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.404+0000 7fd82ca13500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:20.406 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.404+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.406 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.404+0000 7fd82ca13500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:20.406 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.404+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.407 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.404+0000 7fd82ca13500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:20.407 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.404+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.407 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.405+0000 7fd82ca13500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:20.407 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.405+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.407 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.405+0000 7fd82ca13500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.407 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.405+0000 7fd82ca13500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:20.407 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.405+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.409 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.405+0000 7fd82ca13500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.409 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.405+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.409 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.405+0000 7fd82ca13500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:20.409 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.405+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.409 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.406+0000 7fd82ca13500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:20.409 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.406+0000 7fd82ca13500 20 zonegroup default 2023-09-22T13:38:20.409 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.406+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.409 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.406+0000 7fd82ca13500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.409 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.406+0000 7fd82ca13500 10 Cannot find current period zone using local zone 2023-09-22T13:38:20.409 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.406+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.406+0000 7fd82ca13500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.406+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.406+0000 7fd82ca13500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.406+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.407+0000 7fd82ca13500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.407+0000 7fd82ca13500 20 zone default found 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.407+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.407+0000 7fd82ca13500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.407+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.407+0000 7fd82ca13500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.407+0000 7fd82ca13500 20 started sync module instance, tier type = 2023-09-22T13:38:20.410 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.407+0000 7fd82ca13500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:20.421 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.419+0000 7fd82ca13500 20 add_watcher() i=0 2023-09-22T13:38:20.421 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.419+0000 7fd82ca13500 20 add_watcher() i=1 2023-09-22T13:38:20.421 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.419+0000 7fd82ca13500 20 add_watcher() i=2 2023-09-22T13:38:20.421 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.419+0000 7fd82ca13500 20 add_watcher() i=3 2023-09-22T13:38:20.421 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.419+0000 7fd82ca13500 20 add_watcher() i=4 2023-09-22T13:38:20.421 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.419+0000 7fd82ca13500 20 add_watcher() i=5 2023-09-22T13:38:20.421 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.419+0000 7fd82ca13500 20 add_watcher() i=6 2023-09-22T13:38:20.423 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.422+0000 7fd82ca13500 20 add_watcher() i=7 2023-09-22T13:38:20.423 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.422+0000 7fd82ca13500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:20.426 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.425+0000 7fd8186f6700 0 -- 172.21.15.69:0/1642345651 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x55c3493ad150 msgr2=0x55c3493af540 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:20.426 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.425+0000 7fd7e97e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:20.427 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.425+0000 7fd82ca13500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:20.427 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.425+0000 7fd82ca13500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:20.431 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.430+0000 7fd8176f4700 0 -- 172.21.15.69:0/2155830482 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x55c3493f9570 msgr2=0x55c349406020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:38:20.434 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.432+0000 7fd82ca13500 5 note: GC not initialized 2023-09-22T13:38:20.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.433+0000 7fd74ffff700 20 reqs_thread_entry: start 2023-09-22T13:38:20.436 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.433+0000 7fd74dffb700 20 reqs_thread_entry: start 2023-09-22T13:38:20.436 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.433+0000 7fd82ca13500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:20.436 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.433+0000 7fd82ca13500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:20.436 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.433+0000 7fd73ffff700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:50 2023 2023-09-22T13:38:20.436 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:20.436 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd73f7fe700 20 reqs_thread_entry: start 2023-09-22T13:38:20.436 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 20 get_system_obj_state: rctx=0x7fff3e4cae88 obj=default.rgw.meta:root:check-bucket state=0x55c34935f320 s->prefetch_data=0 2023-09-22T13:38:20.436 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:38:20.436 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:38:20.436 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:20.437 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:20.437 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:20.437 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:38:20.437 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.439 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:20.439 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:38:20.439 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:20.439 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 20 get_system_obj_state: rctx=0x7fff3e4caf58 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c34935f320 s->prefetch_data=0 2023-09-22T13:38:20.439 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.434+0000 7fd82ca13500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : miss 2023-09-22T13:38:20.439 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.435+0000 7fd82ca13500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x16 2023-09-22T13:38:20.439 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.435+0000 7fd82ca13500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:20.439 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.435+0000 7fd82ca13500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:20.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.435+0000 7fd82ca13500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:20.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.435+0000 7fd82ca13500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:20.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.435+0000 7fd82ca13500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:20.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.435+0000 7fd82ca13500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:20.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.435+0000 7fd82ca13500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:38:20.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.435+0000 7fd82ca13500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.436+0000 7fd82ca13500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:20.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.436+0000 7fd82ca13500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x13 2023-09-22T13:38:20.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.436+0000 7fd82ca13500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:20.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.436+0000 7fd82ca13500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:20.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.436+0000 7fd82ca13500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:20.441 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.436+0000 7fd82ca13500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 2023-09-22T13:38:20.441 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.436+0000 7fd82ca13500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:20.441 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.436+0000 7fd82ca13500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:38:20.441 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.436+0000 7fd82ca13500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:20.441 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.437+0000 7fd82ca13500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:20.442 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.438+0000 7fd82ca13500 1 NOTICE: finished shard 2 (0 entries found) 2023-09-22T13:38:20.442 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.438+0000 7fd82ca13500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:20.442 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.438+0000 7fd82ca13500 1 NOTICE: finished shard 3 (0 entries found) 2023-09-22T13:38:20.442 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.438+0000 7fd82ca13500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:20.442 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.439+0000 7fd82ca13500 1 NOTICE: finished shard 4 (0 entries found) 2023-09-22T13:38:20.442 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.439+0000 7fd82ca13500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:20.442 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.439+0000 7fd82ca13500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:20.442 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.440+0000 7fd82ca13500 1 NOTICE: finished shard 6 (0 entries found) 2023-09-22T13:38:20.442 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.440+0000 7fd82ca13500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:20.443 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.440+0000 7fd82ca13500 1 NOTICE: finished shard 7 (0 entries found) 2023-09-22T13:38:20.443 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.441+0000 7fd82ca13500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:20.443 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.441+0000 7fd82ca13500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:20.443 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.442+0000 7fd82ca13500 1 NOTICE: finished shard 9 (0 entries found) 2023-09-22T13:38:20.444 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.442+0000 7fd82ca13500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:20.444 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.442+0000 7fd82ca13500 1 NOTICE: finished shard 1 (1 entries found) 2023-09-22T13:38:20.444 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.442+0000 7fd82ca13500 1 NOTICE: finished shard 5 (1 entries found) 2023-09-22T13:38:20.444 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.442+0000 7fd82ca13500 1 NOTICE: finished shard 8 (1 entries found) 2023-09-22T13:38:20.445 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.442+0000 7fd82ca13500 1 NOTICE: finished shard 10 (1 entries found) 2023-09-22T13:38:20.445 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.442+0000 7fd82ca13500 1 NOTICE: finished all shards (4 entries found) 2023-09-22T13:38:20.445 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.444+0000 7fd82ca13500 20 remove_watcher() i=0 2023-09-22T13:38:20.445 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.444+0000 7fd82ca13500 2 removed watcher, disabling cache 2023-09-22T13:38:20.446 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.445+0000 7fd82ca13500 20 remove_watcher() i=1 2023-09-22T13:38:20.449 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.448+0000 7fd82ca13500 20 remove_watcher() i=2 2023-09-22T13:38:20.451 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.449+0000 7fd82ca13500 20 remove_watcher() i=3 2023-09-22T13:38:20.452 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.450+0000 7fd82ca13500 20 remove_watcher() i=4 2023-09-22T13:38:20.453 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.451+0000 7fd82ca13500 20 remove_watcher() i=5 2023-09-22T13:38:20.454 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.452+0000 7fd82ca13500 20 remove_watcher() i=6 2023-09-22T13:38:20.455 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.453+0000 7fd82ca13500 20 remove_watcher() i=7 2023-09-22T13:38:20.455 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.454+0000 7fd7e97e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:20.461 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: unlinked entries are not listable 2023-09-22T13:38:20.461 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:20.469 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:20.469 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:20.470 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: GET returns 404 for unlinked entry keys that have no other versions 2023-09-22T13:38:20.470 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:20.571 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:20.572 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:20.572 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:20.572 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:20.572 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: bucket check unlinked fixes unlistable entries 2023-09-22T13:38:20.572 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:20.573 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys 2023-09-22T13:38:20.601 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:20.601 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:20.609 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.607+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.610 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.609+0000 7f249b7ee500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.611 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.609+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.611 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.610+0000 7f249b7ee500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.611 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.610+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.611 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.610+0000 7f249b7ee500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.611 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.610+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.612 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.611+0000 7f249b7ee500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:20.623 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.621+0000 7f249b7ee500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:20.623 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.621+0000 7f249b7ee500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:20.623 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.621+0000 7f249b7ee500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:20.623 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.621+0000 7f249b7ee500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:20.623 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.621+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.624 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.622+0000 7f249b7ee500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:20.624 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.622+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.624 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.622+0000 7f249b7ee500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:20.625 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.622+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.625 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.623+0000 7f249b7ee500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:20.625 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.623+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.626 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.623+0000 7f249b7ee500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:20.626 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.623+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.627 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.623+0000 7f249b7ee500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.627 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.623+0000 7f249b7ee500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:20.627 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.623+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.627 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.624+0000 7f249b7ee500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.627 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.624+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.627 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.624+0000 7f249b7ee500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:20.627 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.624+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.627 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.624+0000 7f249b7ee500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:20.627 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.624+0000 7f249b7ee500 20 zonegroup default 2023-09-22T13:38:20.627 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.624+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.627 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.624+0000 7f249b7ee500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.628 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.624+0000 7f249b7ee500 10 Cannot find current period zone using local zone 2023-09-22T13:38:20.628 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.624+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.628 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.625+0000 7f249b7ee500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.628 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.625+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.628 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.625+0000 7f249b7ee500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:20.628 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.625+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.629 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.625+0000 7f249b7ee500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:20.629 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.625+0000 7f249b7ee500 20 zone default found 2023-09-22T13:38:20.629 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.625+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.629 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.626+0000 7f249b7ee500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.629 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.626+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.629 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.626+0000 7f249b7ee500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.629 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.626+0000 7f249b7ee500 20 started sync module instance, tier type = 2023-09-22T13:38:20.630 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.626+0000 7f249b7ee500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:20.642 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.640+0000 7f249b7ee500 20 add_watcher() i=0 2023-09-22T13:38:20.642 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.640+0000 7f249b7ee500 20 add_watcher() i=1 2023-09-22T13:38:20.642 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.640+0000 7f249b7ee500 20 add_watcher() i=2 2023-09-22T13:38:20.642 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.640+0000 7f249b7ee500 20 add_watcher() i=3 2023-09-22T13:38:20.642 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.640+0000 7f249b7ee500 20 add_watcher() i=4 2023-09-22T13:38:20.642 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.640+0000 7f249b7ee500 20 add_watcher() i=5 2023-09-22T13:38:20.643 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.640+0000 7f2486cd0700 0 -- 172.21.15.69:0/1715584707 >> [v2:172.21.15.69:6800/145165,v1:172.21.15.69:6801/145165] conn(0x7f247802e390 msgr2=0x7f247802e7d0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:38:20.645 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.643+0000 7f249b7ee500 20 add_watcher() i=6 2023-09-22T13:38:20.645 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.643+0000 7f249b7ee500 20 add_watcher() i=7 2023-09-22T13:38:20.645 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.643+0000 7f249b7ee500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:20.647 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.646+0000 7f2457fe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:20.648 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.646+0000 7f249b7ee500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:20.648 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.646+0000 7f249b7ee500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:20.651 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.650+0000 7f24874d1700 0 -- 172.21.15.69:0/2593902915 >> [v2:172.21.15.69:3301/0,v1:172.21.15.69:6790/0] conn(0x55b1746befe0 msgr2=0x55b1746bf4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:20.852 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.851+0000 7f24874d1700 0 --2- 172.21.15.69:0/2593902915 >> [v2:172.21.15.69:3301/0,v1:172.21.15.69:6790/0] conn(0x55b1746befe0 0x55b1746bf4c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:38:20.854 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.853+0000 7f249b7ee500 5 note: GC not initialized 2023-09-22T13:38:20.856 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.854+0000 7f23c6ffd700 20 reqs_thread_entry: start 2023-09-22T13:38:20.856 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.854+0000 7f23c4ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:20.856 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.854+0000 7f249b7ee500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:20.856 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.854+0000 7f249b7ee500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:20.856 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.854+0000 7f23b6ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:51 2023 2023-09-22T13:38:20.856 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:20.856 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.854+0000 7f23b67fc700 20 reqs_thread_entry: start 2023-09-22T13:38:20.856 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.854+0000 7f249b7ee500 20 get_system_obj_state: rctx=0x7ffc08a388b8 obj=default.rgw.meta:root:check-bucket state=0x55b1746312d0 s->prefetch_data=0 2023-09-22T13:38:20.856 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.854+0000 7f249b7ee500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:38:20.857 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.854+0000 7f249b7ee500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:38:20.858 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:20.858 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:20.858 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:20.858 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:38:20.858 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.858 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:20.858 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:38:20.858 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:20.858 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 20 get_system_obj_state: rctx=0x7ffc08a38988 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55b1746312d0 s->prefetch_data=0 2023-09-22T13:38:20.858 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : miss 2023-09-22T13:38:20.859 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x16 2023-09-22T13:38:20.859 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:20.859 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:20.859 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:20.859 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:20.859 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:20.859 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:20.859 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:38:20.859 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.855+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.859 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.856+0000 7f249b7ee500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:20.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.856+0000 7f249b7ee500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x13 2023-09-22T13:38:20.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.856+0000 7f249b7ee500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:20.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.856+0000 7f249b7ee500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:20.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.856+0000 7f249b7ee500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:20.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.856+0000 7f249b7ee500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 2023-09-22T13:38:20.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.856+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:20.861 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.857+0000 7f249b7ee500 1 NOTICE: finished shard 0 (0 entries removed) 2023-09-22T13:38:20.861 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.857+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:20.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.857+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:20.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.857+0000 7f249b7ee500 1 NOTICE: finished shard 2 (0 entries removed) 2023-09-22T13:38:20.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.858+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:20.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.858+0000 7f249b7ee500 1 NOTICE: finished shard 3 (0 entries removed) 2023-09-22T13:38:20.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.858+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:20.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.858+0000 7f249b7ee500 1 NOTICE: finished shard 4 (0 entries removed) 2023-09-22T13:38:20.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.858+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:20.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.859+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:20.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.859+0000 7f249b7ee500 1 NOTICE: finished shard 6 (0 entries removed) 2023-09-22T13:38:20.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.859+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:20.863 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.860+0000 7f249b7ee500 1 NOTICE: finished shard 7 (0 entries removed) 2023-09-22T13:38:20.863 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.860+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:20.863 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.860+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:20.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.861+0000 7f249b7ee500 1 NOTICE: finished shard 9 (0 entries removed) 2023-09-22T13:38:20.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.861+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:20.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.861+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:20.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.862+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b17475ef70 obj=check-bucket:e state=0x55b174762978 s->prefetch_data=0 2023-09-22T13:38:20.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.862+0000 7f249b7ee500 20 get_obj_state: setting s->obj_tag to kgjufmyn5j0wk1eoemr913xtx69lihst 2023-09-22T13:38:20.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.862+0000 7f249b7ee500 20 get_obj_state_impl: setting s->olh_tag to 67t2j6ogl8worwqx5gvwf7jg0ubvz1zc 2023-09-22T13:38:20.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.862+0000 7f249b7ee500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=67t2j6ogl8worwqx5gvwf7jg0ubvz1zc 2023-09-22T13:38:20.869 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.868+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:20.871 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.869+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:20.871 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.869+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:20.872 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.870+0000 7f249b7ee500 20 olh_log_entry: epoch=2 op=2 key=e[] 2023-09-22T13:38:20.872 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.870+0000 7f249b7ee500 20 olh_log_entry: epoch=2 op=3 key=e[-IzonM1azWwrbAuylGci72akjWmDS4I] 2023-09-22T13:38:20.872 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.870+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b17475ef70 obj=check-bucket:_:-IzonM1azWwrbAuylGci72akjWmDS4I_e state=0x55b1747ccf78 s->prefetch_data=0 2023-09-22T13:38:20.872 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.870+0000 7f249b7ee500 10 manifest: total_size = 22 2023-09-22T13:38:20.872 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.870+0000 7f249b7ee500 20 get_obj_state: setting s->obj_tag to f54d190c-30d0-4b87-88c4-5bcc7de96de4.4553.17582280674926937788 2023-09-22T13:38:20.872 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.870+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b17475ef70 obj=check-bucket:_:-IzonM1azWwrbAuylGci72akjWmDS4I_e state=0x55b1747ccf78 s->prefetch_data=0 2023-09-22T13:38:20.872 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.870+0000 7f249b7ee500 20 prepare_atomic_modification: state is not atomic. state=0x55b1747ccf78 2023-09-22T13:38:20.872 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.870+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:20.878 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : miss 2023-09-22T13:38:20.878 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.879 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.879 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 10 cache put: name=default.rgw.log++bucket.sync-source-hints.check-bucket info.flags=0x0 2023-09-22T13:38:20.879 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 10 adding default.rgw.log++bucket.sync-source-hints.check-bucket to cache LRU end 2023-09-22T13:38:20.879 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : miss 2023-09-22T13:38:20.879 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 20 rados->read ofs=0 len=0 2023-09-22T13:38:20.879 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:20.880 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 10 cache put: name=default.rgw.log++bucket.sync-target-hints.check-bucket info.flags=0x0 2023-09-22T13:38:20.880 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 10 adding default.rgw.log++bucket.sync-target-hints.check-bucket to cache LRU end 2023-09-22T13:38:20.880 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:20.880 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:20.880 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.877+0000 7f249b7ee500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:20.882 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.880+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b17475ef70 obj=check-bucket:e state=0x55b174762978 s->prefetch_data=0 2023-09-22T13:38:20.882 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.880+0000 7f249b7ee500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=67t2j6ogl8worwqx5gvwf7jg0ubvz1zc 2023-09-22T13:38:20.889 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.887+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:20.891 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.888+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:20.891 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.888+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.891 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.888+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.891 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.888+0000 7f249b7ee500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:20.891 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.888+0000 7f249b7ee500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:20.891 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.888+0000 7f249b7ee500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:20.891 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.888+0000 7f249b7ee500 1 NOTICE: finished shard 1 (1 entries removed) 2023-09-22T13:38:20.891 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.888+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:20.891 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.889+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b174783170 obj=check-bucket:d state=0x55b1747628b8 s->prefetch_data=0 2023-09-22T13:38:20.892 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.889+0000 7f249b7ee500 20 get_obj_state: setting s->obj_tag to 86pbql7wcvty2arzkm51jf0xjvyeipqk 2023-09-22T13:38:20.892 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.889+0000 7f249b7ee500 20 get_obj_state_impl: setting s->olh_tag to 8uvw5nsmbksrlwlvxdhz8kel1i52srpn 2023-09-22T13:38:20.892 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.889+0000 7f249b7ee500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=8uvw5nsmbksrlwlvxdhz8kel1i52srpn 2023-09-22T13:38:20.892 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.890+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:20.895 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.893+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:20.895 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.893+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:20.896 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.894+0000 7f249b7ee500 20 olh_log_entry: epoch=3 op=3 key=d[cNXmLY8YkEgJurNcCyGikBlsdhIA33J] 2023-09-22T13:38:20.896 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.894+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b174783170 obj=check-bucket:_:cNXmLY8YkEgJurNcCyGikBlsdhIA33J_d state=0x55b1747ccf78 s->prefetch_data=0 2023-09-22T13:38:20.896 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.894+0000 7f249b7ee500 10 manifest: total_size = 22 2023-09-22T13:38:20.896 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.894+0000 7f249b7ee500 20 get_obj_state: setting s->obj_tag to f54d190c-30d0-4b87-88c4-5bcc7de96de4.4553.10748807195525586689 2023-09-22T13:38:20.896 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.894+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b174783170 obj=check-bucket:_:cNXmLY8YkEgJurNcCyGikBlsdhIA33J_d state=0x55b1747ccf78 s->prefetch_data=0 2023-09-22T13:38:20.896 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.894+0000 7f249b7ee500 20 prepare_atomic_modification: state is not atomic. state=0x55b1747ccf78 2023-09-22T13:38:20.896 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.894+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:20.901 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.900+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.901 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.900+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.901 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.900+0000 7f249b7ee500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:20.901 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.900+0000 7f249b7ee500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:20.901 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.900+0000 7f249b7ee500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:20.908 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.906+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:20.908 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.906+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:20.911 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.909+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:20.911 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.909+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.911 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.909+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.911 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.909+0000 7f249b7ee500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:20.911 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.909+0000 7f249b7ee500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:20.911 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.909+0000 7f249b7ee500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:20.912 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.909+0000 7f249b7ee500 1 NOTICE: finished shard 5 (1 entries removed) 2023-09-22T13:38:20.912 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.909+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:20.912 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.909+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b1747a3b70 obj=check-bucket:c state=0x55b1747628b8 s->prefetch_data=0 2023-09-22T13:38:20.912 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.910+0000 7f249b7ee500 20 get_obj_state: setting s->obj_tag to tsnskpme9z5ope57cmcdux964t2uw5kt 2023-09-22T13:38:20.912 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.910+0000 7f249b7ee500 20 get_obj_state_impl: setting s->olh_tag to 0y9jvly7wks2f8e9spoz93n8qhnbzeoc 2023-09-22T13:38:20.912 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.910+0000 7f249b7ee500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=0y9jvly7wks2f8e9spoz93n8qhnbzeoc 2023-09-22T13:38:20.913 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.912+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:20.917 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.915+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:20.917 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.915+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:20.917 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.915+0000 7f249b7ee500 20 olh_log_entry: epoch=3 op=3 key=c[l-4Mphleajo8c0JGWmg3K.FnYgzXm0T] 2023-09-22T13:38:20.917 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.915+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b1747a3b70 obj=check-bucket:_:l-4Mphleajo8c0JGWmg3K.FnYgzXm0T_c state=0x55b1747ccf78 s->prefetch_data=0 2023-09-22T13:38:20.918 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.916+0000 7f249b7ee500 10 manifest: total_size = 22 2023-09-22T13:38:20.918 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.916+0000 7f249b7ee500 20 get_obj_state: setting s->obj_tag to f54d190c-30d0-4b87-88c4-5bcc7de96de4.4553.18011278065762164608 2023-09-22T13:38:20.918 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.916+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b1747a3b70 obj=check-bucket:_:l-4Mphleajo8c0JGWmg3K.FnYgzXm0T_c state=0x55b1747ccf78 s->prefetch_data=0 2023-09-22T13:38:20.918 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.916+0000 7f249b7ee500 20 prepare_atomic_modification: state is not atomic. state=0x55b1747ccf78 2023-09-22T13:38:20.918 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.916+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:20.921 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.919+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.921 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.919+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.921 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.919+0000 7f249b7ee500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:20.921 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.919+0000 7f249b7ee500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:20.921 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.919+0000 7f249b7ee500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:20.924 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.923+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:20.924 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.923+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:20.928 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.926+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:20.928 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.926+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.928 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.926+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.928 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.926+0000 7f249b7ee500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:20.928 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.926+0000 7f249b7ee500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:20.928 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.926+0000 7f249b7ee500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:20.928 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.926+0000 7f249b7ee500 1 NOTICE: finished shard 8 (1 entries removed) 2023-09-22T13:38:20.928 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.926+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:20.928 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.926+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b1747c4570 obj=check-bucket:f state=0x55b1747628b8 s->prefetch_data=0 2023-09-22T13:38:20.929 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.927+0000 7f249b7ee500 20 get_obj_state: setting s->obj_tag to z0rr5p5cfnm42dwht20kiojp7nwqaqf8 2023-09-22T13:38:20.929 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.927+0000 7f249b7ee500 20 get_obj_state_impl: setting s->olh_tag to pirw3i2uhe3ytgvbg1kgmcal4nq1rm1k 2023-09-22T13:38:20.929 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.927+0000 7f249b7ee500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=pirw3i2uhe3ytgvbg1kgmcal4nq1rm1k 2023-09-22T13:38:20.929 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.928+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:20.931 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.929+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:20.931 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.929+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:20.931 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.929+0000 7f249b7ee500 20 olh_log_entry: epoch=2 op=2 key=f[] 2023-09-22T13:38:20.932 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.930+0000 7f249b7ee500 20 olh_log_entry: epoch=2 op=3 key=f[Nb6YO1RccinBSXq3TcNNos.qUwMqv6W] 2023-09-22T13:38:20.932 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.930+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b1747c4570 obj=check-bucket:_:Nb6YO1RccinBSXq3TcNNos.qUwMqv6W_f state=0x55b1747ccf78 s->prefetch_data=0 2023-09-22T13:38:20.932 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.930+0000 7f249b7ee500 10 manifest: total_size = 12 2023-09-22T13:38:20.932 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.930+0000 7f249b7ee500 20 get_obj_state: setting s->obj_tag to f54d190c-30d0-4b87-88c4-5bcc7de96de4.4553.4810794827942748451 2023-09-22T13:38:20.932 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.930+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b1747c4570 obj=check-bucket:_:Nb6YO1RccinBSXq3TcNNos.qUwMqv6W_f state=0x55b1747ccf78 s->prefetch_data=0 2023-09-22T13:38:20.932 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.930+0000 7f249b7ee500 20 prepare_atomic_modification: state is not atomic. state=0x55b1747ccf78 2023-09-22T13:38:20.932 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.930+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:20.936 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.935+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.936 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.935+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.936 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.935+0000 7f249b7ee500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:20.936 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.935+0000 7f249b7ee500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:20.937 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.935+0000 7f249b7ee500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:20.939 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.937+0000 7f249b7ee500 20 get_obj_state: rctx=0x55b1747c4570 obj=check-bucket:f state=0x55b1747628b8 s->prefetch_data=0 2023-09-22T13:38:20.939 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.937+0000 7f249b7ee500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=pirw3i2uhe3ytgvbg1kgmcal4nq1rm1k 2023-09-22T13:38:20.950 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.948+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:20.951 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.949+0000 7f249b7ee500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:20.951 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.949+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.951 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.949+0000 7f249b7ee500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:20.951 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.949+0000 7f249b7ee500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:20.951 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.949+0000 7f249b7ee500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:20.952 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.949+0000 7f249b7ee500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:20.952 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.949+0000 7f249b7ee500 1 NOTICE: finished shard 10 (1 entries removed) 2023-09-22T13:38:20.952 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.949+0000 7f249b7ee500 1 NOTICE: finished all shards (4 entries removed) 2023-09-22T13:38:20.952 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.951+0000 7f249b7ee500 20 remove_watcher() i=0 2023-09-22T13:38:20.952 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.951+0000 7f249b7ee500 2 removed watcher, disabling cache 2023-09-22T13:38:20.954 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.952+0000 7f249b7ee500 20 remove_watcher() i=1 2023-09-22T13:38:20.957 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.955+0000 7f249b7ee500 20 remove_watcher() i=2 2023-09-22T13:38:20.958 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.956+0000 7f249b7ee500 20 remove_watcher() i=3 2023-09-22T13:38:20.959 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.957+0000 7f249b7ee500 20 remove_watcher() i=4 2023-09-22T13:38:20.960 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.958+0000 7f249b7ee500 20 remove_watcher() i=5 2023-09-22T13:38:20.961 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.959+0000 7f249b7ee500 20 remove_watcher() i=6 2023-09-22T13:38:20.962 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.960+0000 7f249b7ee500 20 remove_watcher() i=7 2023-09-22T13:38:20.963 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:20.961+0000 7f2457fe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:20.978 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:20.978 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:20.978 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:20.979 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:20.979 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:20.979 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:20.979 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:20.979 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:20.979 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: bucket check unlinked does not affect normal entries 2023-09-22T13:38:20.979 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.034 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:21.034 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:21.034 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:21.035 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:21.035 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:21.035 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:21.035 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:21.035 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:21.035 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:21.035 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:21.035 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: bucket check unlinked does not find new unlistable entries after fix 2023-09-22T13:38:21.035 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.036 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --min-age-hours 0 --dump-keys 2023-09-22T13:38:21.062 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:21.062 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:21.070 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.069+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.073 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.071+0000 7fb7ce4da500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.073 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.071+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.074 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.071+0000 7fb7ce4da500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.074 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.071+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.074 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.071+0000 7fb7ce4da500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.074 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.071+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.074 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.072+0000 7fb7ce4da500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.085 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.083+0000 7fb7ce4da500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:21.085 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.083+0000 7fb7ce4da500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:21.085 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.083+0000 7fb7ce4da500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:21.086 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.083+0000 7fb7ce4da500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:21.086 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.083+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.086 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.084+0000 7fb7ce4da500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.086 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.084+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.088 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.084+0000 7fb7ce4da500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.088 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.084+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.088 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.084+0000 7fb7ce4da500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.088 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.084+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.088 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.085+0000 7fb7ce4da500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.088 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.085+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.088 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.085+0000 7fb7ce4da500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.088 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.085+0000 7fb7ce4da500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:21.088 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.085+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.088 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.085+0000 7fb7ce4da500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.089 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.085+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.089 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.085+0000 7fb7ce4da500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.089 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.085+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.089 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.085+0000 7fb7ce4da500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.089 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.085+0000 7fb7ce4da500 20 zonegroup default 2023-09-22T13:38:21.089 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.086+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.089 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.086+0000 7fb7ce4da500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.089 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.086+0000 7fb7ce4da500 10 Cannot find current period zone using local zone 2023-09-22T13:38:21.089 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.086+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.089 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.086+0000 7fb7ce4da500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.089 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.086+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.090 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.086+0000 7fb7ce4da500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.090 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.086+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.091 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.086+0000 7fb7ce4da500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.091 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.087+0000 7fb7ce4da500 20 zone default found 2023-09-22T13:38:21.091 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.087+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.091 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.087+0000 7fb7ce4da500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.091 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.087+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.091 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.087+0000 7fb7ce4da500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.091 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.087+0000 7fb7ce4da500 20 started sync module instance, tier type = 2023-09-22T13:38:21.091 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.087+0000 7fb7ce4da500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:21.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.099+0000 7fb7ce4da500 20 add_watcher() i=0 2023-09-22T13:38:21.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.099+0000 7fb7ce4da500 20 add_watcher() i=1 2023-09-22T13:38:21.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.099+0000 7fb7ce4da500 20 add_watcher() i=2 2023-09-22T13:38:21.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.099+0000 7fb7ce4da500 20 add_watcher() i=3 2023-09-22T13:38:21.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.099+0000 7fb7ce4da500 20 add_watcher() i=4 2023-09-22T13:38:21.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.099+0000 7fb7ce4da500 20 add_watcher() i=5 2023-09-22T13:38:21.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.099+0000 7fb7ce4da500 20 add_watcher() i=6 2023-09-22T13:38:21.103 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.102+0000 7fb7ce4da500 20 add_watcher() i=7 2023-09-22T13:38:21.103 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.102+0000 7fb7ce4da500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:21.104 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.102+0000 7fb7b91bb700 0 -- 172.21.15.69:0/3700392756 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x557367497d20 msgr2=0x55736749a110 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.106 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.105+0000 7fb78afe5700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.106 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.105+0000 7fb7ce4da500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:21.107 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.105+0000 7fb7ce4da500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:21.111 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.109+0000 7fb7b91bb700 0 -- 172.21.15.69:0/688503727 >> [v2:172.21.15.69:3301/0,v1:172.21.15.69:6790/0] conn(0x5573674dd4e0 msgr2=0x5573674e1680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.113 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.112+0000 7fb7ce4da500 5 note: GC not initialized 2023-09-22T13:38:21.114 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.112+0000 7fb6f9ffb700 20 reqs_thread_entry: start 2023-09-22T13:38:21.115 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb6eb7fe700 20 reqs_thread_entry: start 2023-09-22T13:38:21.115 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb7ce4da500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:21.115 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb7ce4da500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:21.115 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb6e9ffb700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:51 2023 2023-09-22T13:38:21.115 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.115 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb6e97fa700 20 reqs_thread_entry: start 2023-09-22T13:38:21.115 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb7ce4da500 20 get_system_obj_state: rctx=0x7ffe73ce96c8 obj=default.rgw.meta:root:check-bucket state=0x5573674db830 s->prefetch_data=0 2023-09-22T13:38:21.115 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb7ce4da500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:38:21.117 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb7ce4da500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:38:21.117 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb7ce4da500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:21.117 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb7ce4da500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:21.117 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb7ce4da500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.117 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb7ce4da500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:38:21.117 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.113+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 20 get_system_obj_state: rctx=0x7ffe73ce9798 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x5573674db830 s->prefetch_data=0 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : miss 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x16 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.118 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.119 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:38:21.119 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.114+0000 7fb7ce4da500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.119 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.115+0000 7fb7ce4da500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.119 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.115+0000 7fb7ce4da500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x13 2023-09-22T13:38:21.119 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.115+0000 7fb7ce4da500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:21.119 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.115+0000 7fb7ce4da500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:21.119 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.115+0000 7fb7ce4da500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:21.119 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.115+0000 7fb7ce4da500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 2023-09-22T13:38:21.119 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.115+0000 7fb7ce4da500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:21.119 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.115+0000 7fb7ce4da500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:38:21.119 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.115+0000 7fb7ce4da500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:21.120 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.116+0000 7fb7ce4da500 1 NOTICE: finished shard 1 (0 entries found) 2023-09-22T13:38:21.120 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.116+0000 7fb7ce4da500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:21.120 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.116+0000 7fb7ce4da500 1 NOTICE: finished shard 2 (0 entries found) 2023-09-22T13:38:21.120 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.116+0000 7fb7ce4da500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:21.120 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.117+0000 7fb7ce4da500 1 NOTICE: finished shard 3 (0 entries found) 2023-09-22T13:38:21.121 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.117+0000 7fb7ce4da500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:21.121 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.118+0000 7fb7ce4da500 1 NOTICE: finished shard 4 (0 entries found) 2023-09-22T13:38:21.121 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.118+0000 7fb7ce4da500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:21.121 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.118+0000 7fb7ce4da500 1 NOTICE: finished shard 5 (0 entries found) 2023-09-22T13:38:21.121 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.118+0000 7fb7ce4da500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:21.121 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.119+0000 7fb7ce4da500 1 NOTICE: finished shard 6 (0 entries found) 2023-09-22T13:38:21.121 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.119+0000 7fb7ce4da500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:21.122 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.119+0000 7fb7ce4da500 1 NOTICE: finished shard 7 (0 entries found) 2023-09-22T13:38:21.122 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.119+0000 7fb7ce4da500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:21.122 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.120+0000 7fb7ce4da500 1 NOTICE: finished shard 8 (0 entries found) 2023-09-22T13:38:21.122 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.120+0000 7fb7ce4da500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:21.122 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.120+0000 7fb7ce4da500 1 NOTICE: finished shard 9 (0 entries found) 2023-09-22T13:38:21.122 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.120+0000 7fb7ce4da500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:21.123 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.120+0000 7fb7ce4da500 1 NOTICE: finished shard 10 (0 entries found) 2023-09-22T13:38:21.123 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.120+0000 7fb7ce4da500 1 NOTICE: finished all shards (0 entries found) 2023-09-22T13:38:21.123 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.122+0000 7fb7ce4da500 20 remove_watcher() i=0 2023-09-22T13:38:21.123 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.122+0000 7fb7ce4da500 2 removed watcher, disabling cache 2023-09-22T13:38:21.125 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.123+0000 7fb7ce4da500 20 remove_watcher() i=1 2023-09-22T13:38:21.130 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.129+0000 7fb7ce4da500 20 remove_watcher() i=2 2023-09-22T13:38:21.131 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.130+0000 7fb7ce4da500 20 remove_watcher() i=3 2023-09-22T13:38:21.132 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.131+0000 7fb7ce4da500 20 remove_watcher() i=4 2023-09-22T13:38:21.133 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.132+0000 7fb7ce4da500 20 remove_watcher() i=5 2023-09-22T13:38:21.135 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.133+0000 7fb7ce4da500 20 remove_watcher() i=6 2023-09-22T13:38:21.136 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.134+0000 7fb7ce4da500 20 remove_watcher() i=7 2023-09-22T13:38:21.136 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.135+0000 7fb78afe5700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.177 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:21.177 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:21.177 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:21.177 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:21.177 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:38:21.201 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:21.201 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:21.208 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.207+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.210 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.209+0000 7f1d17b81500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.210 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.209+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.211 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.209+0000 7f1d17b81500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.211 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.209+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.211 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.209+0000 7f1d17b81500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.211 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.209+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.212 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.210+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.222 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.220+0000 7f1d17b81500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:21.222 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.220+0000 7f1d17b81500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:21.222 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.220+0000 7f1d17b81500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:21.222 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.220+0000 7f1d17b81500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:21.222 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.220+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.222 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.220+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.222 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.220+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.224 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.221+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.224 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.221+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.224 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.221+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.224 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.221+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.224 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.221+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.224 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.221+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.224 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.222+0000 7f1d17b81500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.224 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.222+0000 7f1d17b81500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:21.224 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.222+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.224 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.222+0000 7f1d17b81500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.225 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.222+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.225 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.222+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.225 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.222+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.226 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.222+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.226 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.223+0000 7f1d17b81500 20 zonegroup default 2023-09-22T13:38:21.226 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.223+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.226 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.223+0000 7f1d17b81500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.226 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.223+0000 7f1d17b81500 10 Cannot find current period zone using local zone 2023-09-22T13:38:21.226 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.223+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.226 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.223+0000 7f1d17b81500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.226 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.223+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.227 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.223+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.227 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.223+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.227 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.224+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.227 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.224+0000 7f1d17b81500 20 zone default found 2023-09-22T13:38:21.227 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.224+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.227 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.224+0000 7f1d17b81500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.227 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.224+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.227 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.225+0000 7f1d17b81500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.227 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.225+0000 7f1d17b81500 20 started sync module instance, tier type = 2023-09-22T13:38:21.227 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.225+0000 7f1d17b81500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:21.237 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.235+0000 7f1d17b81500 20 add_watcher() i=0 2023-09-22T13:38:21.237 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.235+0000 7f1d17b81500 20 add_watcher() i=1 2023-09-22T13:38:21.237 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.235+0000 7f1d17b81500 20 add_watcher() i=2 2023-09-22T13:38:21.237 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.235+0000 7f1d17b81500 20 add_watcher() i=3 2023-09-22T13:38:21.237 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.235+0000 7f1d17b81500 20 add_watcher() i=4 2023-09-22T13:38:21.238 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.235+0000 7f1d17b81500 20 add_watcher() i=5 2023-09-22T13:38:21.238 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.235+0000 7f1d17b81500 20 add_watcher() i=6 2023-09-22T13:38:21.238 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.236+0000 7f1d17b81500 20 add_watcher() i=7 2023-09-22T13:38:21.238 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.236+0000 7f1d17b81500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:21.238 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.237+0000 7f1d03864700 0 -- 172.21.15.69:0/3533689897 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x556b55754af0 msgr2=0x556b55756ee0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:38:21.240 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.238+0000 7f1cd4fe1700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.240 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.239+0000 7f1d17b81500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:21.240 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.239+0000 7f1d17b81500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:21.244 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.243+0000 7f1d03864700 0 -- 172.21.15.69:0/2139809461 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x556b55793f90 msgr2=0x556b55796510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.246 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.245+0000 7f1d17b81500 5 note: GC not initialized 2023-09-22T13:38:21.247 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.246+0000 7f1c437fe700 20 reqs_thread_entry: start 2023-09-22T13:38:21.248 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.246+0000 7f1c417fa700 20 reqs_thread_entry: start 2023-09-22T13:38:21.248 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.246+0000 7f1d17b81500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:21.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.246+0000 7f1d17b81500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:21.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.246+0000 7f1c2f7fe700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:51 2023 2023-09-22T13:38:21.249 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.246+0000 7f1c2effd700 20 reqs_thread_entry: start 2023-09-22T13:38:21.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.247+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b6e0 obj=default.rgw.meta:root:check-bucket state=0x556b55737360 s->prefetch_data=0 2023-09-22T13:38:21.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.247+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.247+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.247+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:21.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.247+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b6e0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.250 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.248+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.250 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.248+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.248+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.248+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.248+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.248+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.248+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b4b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.249+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.249+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.249+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.249+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.249+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.249+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.249+0000 7f1d17b81500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:21.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.249+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b4b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.250+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.250+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.250+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.253 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.250+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.253 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.251+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.253 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.251+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.253 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.251+0000 7f1d17b81500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:21.253 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.251+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b4b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.253 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.252+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.254 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.252+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.254 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.252+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.254 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.252+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.254 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.252+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.254 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.252+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.254 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.252+0000 7f1d17b81500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:21.255 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.253+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b4b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.255 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.253+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.255 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.253+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.255 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.253+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.255 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.253+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.256 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.254+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.256 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.254+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.256 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.254+0000 7f1d17b81500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:21.256 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.254+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b4b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.256 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.254+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.256 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.254+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.257 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.254+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.257 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.254+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.257 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.255+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.257 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.255+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.257 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.255+0000 7f1d17b81500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:21.257 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.255+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b4b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.257 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.256+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.258 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.256+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.258 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.256+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.258 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.256+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.258 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.256+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.258 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.256+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.258 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.256+0000 7f1d17b81500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:21.259 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.257+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b4b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.259 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.257+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.259 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.257+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.259 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.257+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.259 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.257+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.260 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.258+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.260 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.258+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.260 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.258+0000 7f1d17b81500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:21.260 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.258+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b4b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.260 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.259+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.260 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.259+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.261 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.259+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.261 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.259+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.261 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.259+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.261 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.259+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.261 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.259+0000 7f1d17b81500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:21.262 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.260+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b4b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.262 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.260+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.262 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.260+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.262 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.260+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.262 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.260+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.263 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.261+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.263 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.261+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.263 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.261+0000 7f1d17b81500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:21.263 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.261+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b4b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.263 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.261+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.263 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.261+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.263 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.261+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.263 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.261+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.264 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.262+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.264 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.262+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.265 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.262+0000 7f1d17b81500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:21.265 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.262+0000 7f1d17b81500 20 get_system_obj_state: rctx=0x7ffe18b9b4b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x556b555e7810 s->prefetch_data=0 2023-09-22T13:38:21.265 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.262+0000 7f1d17b81500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.265 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.262+0000 7f1d17b81500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.265 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.262+0000 7f1d17b81500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.265 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.262+0000 7f1d17b81500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.265 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.263+0000 7f1d17b81500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.265 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.263+0000 7f1d17b81500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.265 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.263+0000 7f1d17b81500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:21.266 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.264+0000 7f1d17b81500 20 remove_watcher() i=0 2023-09-22T13:38:21.266 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.264+0000 7f1d17b81500 2 removed watcher, disabling cache 2023-09-22T13:38:21.267 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.266+0000 7f1d17b81500 20 remove_watcher() i=1 2023-09-22T13:38:21.268 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.267+0000 7f1d17b81500 20 remove_watcher() i=2 2023-09-22T13:38:21.269 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.268+0000 7f1d17b81500 20 remove_watcher() i=3 2023-09-22T13:38:21.271 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.269+0000 7f1d17b81500 20 remove_watcher() i=4 2023-09-22T13:38:21.271 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.270+0000 7f1d17b81500 20 remove_watcher() i=5 2023-09-22T13:38:21.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.271+0000 7f1d17b81500 20 remove_watcher() i=6 2023-09-22T13:38:21.273 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.272+0000 7f1d17b81500 20 remove_watcher() i=7 2023-09-22T13:38:21.274 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.272+0000 7f1cd4fe1700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.280 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:38:21.305 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:21.305 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:21.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.311+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.314 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.313+0000 7f7d85112500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.314 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.313+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.315 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.313+0000 7f7d85112500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.315 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.313+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.315 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.314+0000 7f7d85112500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.315 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.314+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.316 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.315+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.325+0000 7f7d85112500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:21.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.325+0000 7f7d85112500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:21.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.325+0000 7f7d85112500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:21.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.325+0000 7f7d85112500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:21.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.325+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.325+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.325+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.325+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.325+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.329 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.326+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.326+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.326+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.326+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.326+0000 7f7d85112500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.326+0000 7f7d85112500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:21.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.326+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.326+0000 7f7d85112500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.326+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.327+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.327+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.327+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.327+0000 7f7d85112500 20 zonegroup default 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.327+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.327+0000 7f7d85112500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.327+0000 7f7d85112500 10 Cannot find current period zone using local zone 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.327+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.327+0000 7f7d85112500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.327+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.328+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.328+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.328+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.328+0000 7f7d85112500 20 zone default found 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.328+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.328+0000 7f7d85112500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.332 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.328+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.332 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.328+0000 7f7d85112500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.332 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.328+0000 7f7d85112500 20 started sync module instance, tier type = 2023-09-22T13:38:21.332 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.328+0000 7f7d85112500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:21.339 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.338+0000 7f7d85112500 20 add_watcher() i=0 2023-09-22T13:38:21.339 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.338+0000 7f7d85112500 20 add_watcher() i=1 2023-09-22T13:38:21.339 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.338+0000 7f7d85112500 20 add_watcher() i=2 2023-09-22T13:38:21.339 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.338+0000 7f7d85112500 20 add_watcher() i=3 2023-09-22T13:38:21.340 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.338+0000 7f7d85112500 20 add_watcher() i=4 2023-09-22T13:38:21.340 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.338+0000 7f7d85112500 20 add_watcher() i=5 2023-09-22T13:38:21.342 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.340+0000 7f7d85112500 20 add_watcher() i=6 2023-09-22T13:38:21.342 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.340+0000 7f7d85112500 20 add_watcher() i=7 2023-09-22T13:38:21.342 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.340+0000 7f7d85112500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:21.343 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.342+0000 7f7d41fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.344 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.342+0000 7f7d85112500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:21.344 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.342+0000 7f7d85112500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:21.345 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.343+0000 7f7d6fdf3700 0 -- >> [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] conn(0x55c2f47d7930 msgr2=0x55c2f47d7db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0)._try_send injecting socket failure 2023-09-22T13:38:21.349 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.348+0000 7f7d85112500 5 note: GC not initialized 2023-09-22T13:38:21.350 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.349+0000 7f7cb8ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:21.351 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.349+0000 7f7cae7fc700 20 reqs_thread_entry: start 2023-09-22T13:38:21.351 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.349+0000 7f7d85112500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:21.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.349+0000 7f7d85112500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:21.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.349+0000 7f7cacff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:51 2023 2023-09-22T13:38:21.352 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.349+0000 7f7c93fff700 20 reqs_thread_entry: start 2023-09-22T13:38:21.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.350+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efc1b0 obj=default.rgw.meta:root:check-bucket state=0x55c2f47d77b0 s->prefetch_data=0 2023-09-22T13:38:21.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.350+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.350+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.350+0000 7f7d705f4700 0 -- 172.21.15.69:0/776613118 >> [v2:172.21.15.69:6808/145166,v1:172.21.15.69:6809/145166] conn(0x55c2f479e030 msgr2=0x55c2f47a04b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.350+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:21.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.350+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efc1b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.353 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.351+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.353 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.351+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.353 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.351+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.353 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.351+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.351+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.351+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.351+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efbf80 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.352+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.352+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.352+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.352+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.352+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.352+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.352+0000 7f7d85112500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:21.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.353+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efbf80 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.353+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.353+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.353+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.353+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.354+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.354+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.354+0000 7f7d85112500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:21.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.354+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efbf80 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.354+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.354+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.354+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.354+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.355+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.355+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.355+0000 7f7d85112500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:21.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.355+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efbf80 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.355+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.355+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.355+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.355+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.356+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.356+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.356+0000 7f7d85112500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:21.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.356+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efbf80 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.357+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.357+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.357+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.357+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.357+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.357+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.357+0000 7f7d85112500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:21.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.358+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efbf80 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.358+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.358+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.358+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.358+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.359+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.359+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.359+0000 7f7d85112500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:21.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.359+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efbf80 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.360+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.360+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.360+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.360+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.360+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.360+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.363 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.360+0000 7f7d85112500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:21.363 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.361+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efbf80 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.363 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.361+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.363 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.361+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.361+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.361+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.362+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.362+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.362+0000 7f7d85112500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:21.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.362+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efbf80 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.362+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.362+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.362+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.362+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.363+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.363+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.363+0000 7f7d85112500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:21.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.363+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efbf80 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.363+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.363+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.363+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.363+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.364+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.364+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.364+0000 7f7d85112500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:21.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.364+0000 7f7d85112500 20 get_system_obj_state: rctx=0x7ffc27efbf80 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c2f47e42b0 s->prefetch_data=0 2023-09-22T13:38:21.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.364+0000 7f7d85112500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.364+0000 7f7d85112500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.364+0000 7f7d85112500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.364+0000 7f7d85112500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.365+0000 7f7d85112500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.365+0000 7f7d85112500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.365+0000 7f7d85112500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:21.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.366+0000 7f7d85112500 20 remove_watcher() i=0 2023-09-22T13:38:21.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.366+0000 7f7d85112500 2 removed watcher, disabling cache 2023-09-22T13:38:21.369 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.367+0000 7f7d85112500 20 remove_watcher() i=1 2023-09-22T13:38:21.370 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.368+0000 7f7d85112500 20 remove_watcher() i=2 2023-09-22T13:38:21.371 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.369+0000 7f7d85112500 20 remove_watcher() i=3 2023-09-22T13:38:21.372 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.370+0000 7f7d85112500 20 remove_watcher() i=4 2023-09-22T13:38:21.373 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.371+0000 7f7d85112500 20 remove_watcher() i=5 2023-09-22T13:38:21.375 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.374+0000 7f7d85112500 20 remove_watcher() i=6 2023-09-22T13:38:21.376 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.375+0000 7f7d85112500 20 remove_watcher() i=7 2023-09-22T13:38:21.377 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.376+0000 7f7d41fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.382 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: bucket check olh finds leftover OLH entries 2023-09-22T13:38:21.382 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.383 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bucket check olh --bucket check-bucket --dump-keys 2023-09-22T13:38:21.408 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:21.408 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:21.415 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.414+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.416+0000 7febc0a75500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.416+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.418 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.416+0000 7febc0a75500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.418 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.416+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.418 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.417+0000 7febc0a75500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.419 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.417+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.419 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.418+0000 7febc0a75500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.430 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.428+0000 7febc0a75500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:21.430 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.428+0000 7febc0a75500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:21.430 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.428+0000 7febc0a75500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:21.430 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.428+0000 7febc0a75500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:21.430 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.428+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.431 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.428+0000 7febc0a75500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.431 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.428+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.431 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.429+0000 7febc0a75500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.431 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.429+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.431 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.429+0000 7febc0a75500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.432 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.429+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.432 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.429+0000 7febc0a75500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.432 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.429+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.432 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.429+0000 7febc0a75500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.432 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.429+0000 7febc0a75500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:21.432 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.429+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.432 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.430+0000 7febc0a75500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.432 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.430+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.434 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.430+0000 7febc0a75500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.434 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.430+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.434 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.430+0000 7febc0a75500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.434 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.430+0000 7febc0a75500 20 zonegroup default 2023-09-22T13:38:21.434 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.430+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.434 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.430+0000 7febc0a75500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.434 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.430+0000 7febc0a75500 10 Cannot find current period zone using local zone 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.430+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.431+0000 7febc0a75500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.431+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.431+0000 7febc0a75500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.431+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.431+0000 7febc0a75500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.431+0000 7febc0a75500 20 zone default found 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.431+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.432+0000 7febc0a75500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.432+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.432+0000 7febc0a75500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.432+0000 7febc0a75500 20 started sync module instance, tier type = 2023-09-22T13:38:21.436 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.432+0000 7febc0a75500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:21.452 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.450+0000 7febc0a75500 20 add_watcher() i=0 2023-09-22T13:38:21.452 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.450+0000 7febc0a75500 20 add_watcher() i=1 2023-09-22T13:38:21.452 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.450+0000 7febc0a75500 20 add_watcher() i=2 2023-09-22T13:38:21.452 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.450+0000 7febc0a75500 20 add_watcher() i=3 2023-09-22T13:38:21.452 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.450+0000 7febc0a75500 20 add_watcher() i=4 2023-09-22T13:38:21.452 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.450+0000 7febc0a75500 20 add_watcher() i=5 2023-09-22T13:38:21.452 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.450+0000 7febabf57700 0 -- 172.21.15.69:0/2803931264 >> [v2:172.21.15.69:6808/145166,v1:172.21.15.69:6809/145166] conn(0x55c728b7a200 msgr2=0x55c728b80340 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.453 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.450+0000 7febc0a75500 20 add_watcher() i=6 2023-09-22T13:38:21.453 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.451+0000 7febc0a75500 20 add_watcher() i=7 2023-09-22T13:38:21.453 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.451+0000 7febc0a75500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:21.454 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.453+0000 7feb7d7e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.454 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.453+0000 7febc0a75500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:21.455 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.453+0000 7febc0a75500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:21.458 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.457+0000 7febab756700 0 -- 172.21.15.69:0/1720018650 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x55c728bbeed0 msgr2=0x55c728bb4140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.460 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.459+0000 7febc0a75500 5 note: GC not initialized 2023-09-22T13:38:21.462 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.460+0000 7feaebfff700 20 reqs_thread_entry: start 2023-09-22T13:38:21.463 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.460+0000 7feae9ffb700 20 reqs_thread_entry: start 2023-09-22T13:38:21.463 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.460+0000 7febc0a75500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:21.463 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.460+0000 7febc0a75500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:21.463 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.460+0000 7fead7fff700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:51 2023 2023-09-22T13:38:21.463 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.463 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.460+0000 7fead77fe700 20 reqs_thread_entry: start 2023-09-22T13:38:21.463 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.460+0000 7febc0a75500 20 get_system_obj_state: rctx=0x7ffd2d9285c8 obj=default.rgw.meta:root:check-bucket state=0x55c728a8c0c0 s->prefetch_data=0 2023-09-22T13:38:21.463 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.460+0000 7febc0a75500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:38:21.463 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:38:21.463 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:21.463 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:21.464 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.464 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:38:21.464 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.464 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:21.464 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:38:21.464 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:21.464 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 20 get_system_obj_state: rctx=0x7ffd2d928698 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55c728a8c0c0 s->prefetch_data=0 2023-09-22T13:38:21.464 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : miss 2023-09-22T13:38:21.466 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x16 2023-09-22T13:38:21.466 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:21.466 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:21.466 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:21.466 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.466 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.466 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.466 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:38:21.466 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.461+0000 7febc0a75500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.462+0000 7febc0a75500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.462+0000 7febc0a75500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x13 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.462+0000 7febc0a75500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.462+0000 7febc0a75500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.462+0000 7febc0a75500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.462+0000 7febc0a75500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.462+0000 7febc0a75500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.462+0000 7febc0a75500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.462+0000 7febc0a75500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.463+0000 7febc0a75500 1 NOTICE: finished shard 1 (0 entries found) 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.463+0000 7febc0a75500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.463+0000 7febc0a75500 1 NOTICE: finished shard 2 (0 entries found) 2023-09-22T13:38:21.467 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.463+0000 7febc0a75500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:21.468 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.464+0000 7febc0a75500 1 NOTICE: finished shard 3 (0 entries found) 2023-09-22T13:38:21.468 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.464+0000 7febc0a75500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:21.468 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.464+0000 7febc0a75500 1 NOTICE: finished shard 4 (0 entries found) 2023-09-22T13:38:21.468 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.464+0000 7febc0a75500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:21.469 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.465+0000 7febc0a75500 1 NOTICE: finished shard 5 (1 entries found) 2023-09-22T13:38:21.469 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.465+0000 7febc0a75500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:21.469 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.465+0000 7febc0a75500 1 NOTICE: finished shard 6 (0 entries found) 2023-09-22T13:38:21.469 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.465+0000 7febc0a75500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:21.469 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.465+0000 7febc0a75500 1 NOTICE: finished shard 7 (0 entries found) 2023-09-22T13:38:21.469 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.465+0000 7febc0a75500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:21.469 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.466+0000 7febc0a75500 1 NOTICE: finished shard 8 (1 entries found) 2023-09-22T13:38:21.470 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.466+0000 7febc0a75500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:21.470 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.466+0000 7febc0a75500 1 NOTICE: finished shard 9 (0 entries found) 2023-09-22T13:38:21.470 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.466+0000 7febc0a75500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:21.470 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.467+0000 7febc0a75500 1 NOTICE: finished shard 10 (0 entries found) 2023-09-22T13:38:21.470 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.467+0000 7febc0a75500 1 NOTICE: finished all shards (2 entries found) 2023-09-22T13:38:21.470 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.468+0000 7febc0a75500 20 remove_watcher() i=0 2023-09-22T13:38:21.470 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.468+0000 7febc0a75500 2 removed watcher, disabling cache 2023-09-22T13:38:21.471 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.469+0000 7febc0a75500 20 remove_watcher() i=1 2023-09-22T13:38:21.472 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.470+0000 7febc0a75500 20 remove_watcher() i=2 2023-09-22T13:38:21.473 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.471+0000 7febc0a75500 20 remove_watcher() i=3 2023-09-22T13:38:21.474 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.472+0000 7febc0a75500 20 remove_watcher() i=4 2023-09-22T13:38:21.475 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.474+0000 7febc0a75500 20 remove_watcher() i=5 2023-09-22T13:38:21.478 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.477+0000 7febc0a75500 20 remove_watcher() i=6 2023-09-22T13:38:21.479 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.478+0000 7febc0a75500 20 remove_watcher() i=7 2023-09-22T13:38:21.481 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.478+0000 7feb7d7e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.485 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: bucket check olh fixes leftover OLH entries 2023-09-22T13:38:21.485 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.486 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bucket check olh --bucket check-bucket --fix --rgw-olh-pending-timeout-sec 0 --dump-keys 2023-09-22T13:38:21.513 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:21.513 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:21.521 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.520+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.522 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.521+0000 7fa81d2b7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.522 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.521+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.523 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.522+0000 7fa81d2b7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.523 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.522+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.523 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.522+0000 7fa81d2b7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.523 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.522+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.524 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.523+0000 7fa81d2b7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.534 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.533+0000 7fa81d2b7500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:21.535 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.533+0000 7fa81d2b7500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:21.535 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.533+0000 7fa81d2b7500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:21.535 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.533+0000 7fa81d2b7500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:21.535 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.533+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.536 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.533+0000 7fa81d2b7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.536 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.533+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.536 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.534+0000 7fa81d2b7500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.536 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.534+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.536 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.534+0000 7fa81d2b7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.536 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.534+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.537 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.534+0000 7fa81d2b7500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.537 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.534+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.537 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.535+0000 7fa81d2b7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.537 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.535+0000 7fa81d2b7500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:21.537 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.535+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.538 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.535+0000 7fa81d2b7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.535+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.535+0000 7fa81d2b7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.535+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.535+0000 7fa81d2b7500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.535+0000 7fa81d2b7500 20 zonegroup default 2023-09-22T13:38:21.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.535+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.536+0000 7fa81d2b7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.536+0000 7fa81d2b7500 10 Cannot find current period zone using local zone 2023-09-22T13:38:21.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.536+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.536+0000 7fa81d2b7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.539 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.536+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.536+0000 7fa81d2b7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.536+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.536+0000 7fa81d2b7500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.536+0000 7fa81d2b7500 20 zone default found 2023-09-22T13:38:21.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.536+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.537+0000 7fa81d2b7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.537+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.537+0000 7fa81d2b7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.537+0000 7fa81d2b7500 20 started sync module instance, tier type = 2023-09-22T13:38:21.540 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.537+0000 7fa81d2b7500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:21.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.546+0000 7fa81d2b7500 20 add_watcher() i=0 2023-09-22T13:38:21.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.546+0000 7fa81d2b7500 20 add_watcher() i=1 2023-09-22T13:38:21.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.546+0000 7fa81d2b7500 20 add_watcher() i=2 2023-09-22T13:38:21.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.546+0000 7fa81d2b7500 20 add_watcher() i=3 2023-09-22T13:38:21.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.546+0000 7fa81d2b7500 20 add_watcher() i=4 2023-09-22T13:38:21.548 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.546+0000 7fa81d2b7500 20 add_watcher() i=5 2023-09-22T13:38:21.549 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.546+0000 7fa81d2b7500 20 add_watcher() i=6 2023-09-22T13:38:21.549 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.547+0000 7fa81d2b7500 20 add_watcher() i=7 2023-09-22T13:38:21.549 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.547+0000 7fa81d2b7500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:21.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.549+0000 7fa807f98700 0 -- 172.21.15.69:0/3363344534 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x5626418b52c0 msgr2=0x5626418b76b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.549+0000 7fa7d9fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.550+0000 7fa81d2b7500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:21.551 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.550+0000 7fa81d2b7500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:21.556 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.554+0000 7fa808f9a700 0 -- 172.21.15.69:0/1352557066 >> [v2:172.21.15.69:3301/0,v1:172.21.15.69:6790/0] conn(0x5626418fc4b0 msgr2=0x5626418fac20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.558 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.557+0000 7fa81d2b7500 5 note: GC not initialized 2023-09-22T13:38:21.559 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.557+0000 7fa750ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:21.560 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa7467fc700 20 reqs_thread_entry: start 2023-09-22T13:38:21.560 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa81d2b7500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:21.560 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa81d2b7500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:21.560 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa744ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:51 2023 2023-09-22T13:38:21.560 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.560 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa72bfff700 20 reqs_thread_entry: start 2023-09-22T13:38:21.560 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa81d2b7500 20 get_system_obj_state: rctx=0x7ffd12bfea78 obj=default.rgw.meta:root:check-bucket state=0x5626418d1160 s->prefetch_data=0 2023-09-22T13:38:21.560 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa81d2b7500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:38:21.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa81d2b7500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:38:21.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa81d2b7500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:21.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa81d2b7500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:21.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa81d2b7500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa81d2b7500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:38:21.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.558+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:21.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:38:21.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:21.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 20 get_system_obj_state: rctx=0x7ffd12bfeb48 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x5626418d1160 s->prefetch_data=0 2023-09-22T13:38:21.562 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : miss 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x16 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.559+0000 7fa81d2b7500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.560+0000 7fa81d2b7500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.560+0000 7fa81d2b7500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x13 2023-09-22T13:38:21.563 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.560+0000 7fa81d2b7500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:21.564 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.560+0000 7fa81d2b7500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:21.564 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.560+0000 7fa81d2b7500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:21.564 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.560+0000 7fa81d2b7500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 2023-09-22T13:38:21.564 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.560+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:21.564 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.560+0000 7fa81d2b7500 1 NOTICE: finished shard 0 (0 entries removed) 2023-09-22T13:38:21.564 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.560+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:21.565 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.561+0000 7fa81d2b7500 1 NOTICE: finished shard 1 (0 entries removed) 2023-09-22T13:38:21.565 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.561+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:21.565 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.561+0000 7fa81d2b7500 1 NOTICE: finished shard 2 (0 entries removed) 2023-09-22T13:38:21.565 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.561+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:21.565 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.561+0000 7fa81d2b7500 1 NOTICE: finished shard 3 (0 entries removed) 2023-09-22T13:38:21.566 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.561+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:21.566 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.562+0000 7fa81d2b7500 1 NOTICE: finished shard 4 (0 entries removed) 2023-09-22T13:38:21.566 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.562+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:21.566 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.562+0000 7fa81d2b7500 20 get_obj_state: rctx=0x562641996380 obj=check-bucket:d state=0x562641999698 s->prefetch_data=0 2023-09-22T13:38:21.566 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.562+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:21.566 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.563+0000 7fa81d2b7500 1 NOTICE: finished shard 6 (0 entries removed) 2023-09-22T13:38:21.566 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.563+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:21.566 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.563+0000 7fa81d2b7500 1 NOTICE: finished shard 7 (0 entries removed) 2023-09-22T13:38:21.566 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.563+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:21.566 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.564+0000 7fa81d2b7500 20 get_obj_state: rctx=0x5626419ba180 obj=check-bucket:c state=0x562641997638 s->prefetch_data=0 2023-09-22T13:38:21.566 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.564+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:21.567 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.564+0000 7fa81d2b7500 1 NOTICE: finished shard 9 (0 entries removed) 2023-09-22T13:38:21.567 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.564+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:21.567 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.564+0000 7fa81d2b7500 1 NOTICE: finished shard 10 (0 entries removed) 2023-09-22T13:38:21.567 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.564+0000 7fa81d2b7500 20 get_obj_state: setting s->obj_tag to 86pbql7wcvty2arzkm51jf0xjvyeipqk 2023-09-22T13:38:21.568 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.564+0000 7fa81d2b7500 20 get_obj_state_impl: setting s->olh_tag to 8uvw5nsmbksrlwlvxdhz8kel1i52srpn 2023-09-22T13:38:21.568 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.564+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:21.568 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.564+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:21.568 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.565+0000 7fa81d2b7500 20 olh_log_entry: epoch=4 op=2 key=d[] 2023-09-22T13:38:21.568 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.565+0000 7fa81d2b7500 20 olh_log_entry: epoch=4 op=3 key=d[-9f5AUdhOPSBjpvUBITWCDOu346.bmx] 2023-09-22T13:38:21.568 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.565+0000 7fa81d2b7500 20 get_obj_state: rctx=0x562641996380 obj=check-bucket:_:-9f5AUdhOPSBjpvUBITWCDOu346.bmx_d state=0x5626419c15d8 s->prefetch_data=0 2023-09-22T13:38:21.568 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.567+0000 7fa81d2b7500 20 get_obj_state: rctx=0x562641996380 obj=check-bucket:d state=0x562641999698 s->prefetch_data=0 2023-09-22T13:38:21.568 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.567+0000 7fa81d2b7500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=8uvw5nsmbksrlwlvxdhz8kel1i52srpn 2023-09-22T13:38:21.571 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.570+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:21.575 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.574+0000 7fa81d2b7500 1 NOTICE: finished shard 5 (1 entries removed) 2023-09-22T13:38:21.576 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.574+0000 7fa81d2b7500 20 get_obj_state: setting s->obj_tag to tsnskpme9z5ope57cmcdux964t2uw5kt 2023-09-22T13:38:21.576 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.574+0000 7fa81d2b7500 20 get_obj_state_impl: setting s->olh_tag to 0y9jvly7wks2f8e9spoz93n8qhnbzeoc 2023-09-22T13:38:21.576 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.574+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:21.576 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.574+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:21.576 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.574+0000 7fa81d2b7500 20 olh_log_entry: epoch=4 op=2 key=c[] 2023-09-22T13:38:21.576 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.574+0000 7fa81d2b7500 20 olh_log_entry: epoch=4 op=3 key=c[sTkGfNgQVnDYDFxY8OiHbu8sE.gL8Ff] 2023-09-22T13:38:21.576 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.574+0000 7fa81d2b7500 20 get_obj_state: rctx=0x5626419ba180 obj=check-bucket:_:sTkGfNgQVnDYDFxY8OiHbu8sE.gL8Ff_c state=0x5626419c3188 s->prefetch_data=0 2023-09-22T13:38:21.577 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.576+0000 7fa81d2b7500 20 get_obj_state: rctx=0x5626419ba180 obj=check-bucket:c state=0x562641997638 s->prefetch_data=0 2023-09-22T13:38:21.577 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.576+0000 7fa81d2b7500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=0y9jvly7wks2f8e9spoz93n8qhnbzeoc 2023-09-22T13:38:21.586 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.585+0000 7fa81d2b7500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:21.591 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.590+0000 7fa81d2b7500 1 NOTICE: finished shard 8 (1 entries removed) 2023-09-22T13:38:21.591 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.590+0000 7fa81d2b7500 1 NOTICE: finished all shards (2 entries removed) 2023-09-22T13:38:21.592 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.591+0000 7fa81d2b7500 20 remove_watcher() i=0 2023-09-22T13:38:21.592 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.591+0000 7fa81d2b7500 2 removed watcher, disabling cache 2023-09-22T13:38:21.593 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.592+0000 7fa81d2b7500 20 remove_watcher() i=1 2023-09-22T13:38:21.594 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.593+0000 7fa81d2b7500 20 remove_watcher() i=2 2023-09-22T13:38:21.595 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.594+0000 7fa81d2b7500 20 remove_watcher() i=3 2023-09-22T13:38:21.597 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.595+0000 7fa81d2b7500 20 remove_watcher() i=4 2023-09-22T13:38:21.598 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.596+0000 7fa81d2b7500 20 remove_watcher() i=5 2023-09-22T13:38:21.599 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.598+0000 7fa81d2b7500 20 remove_watcher() i=6 2023-09-22T13:38:21.600 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.599+0000 7fa81d2b7500 20 remove_watcher() i=7 2023-09-22T13:38:21.601 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.600+0000 7fa7d9fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.607 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:38:21.633 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:21.633 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:21.640 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.639+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.642 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.641+0000 7f60c2279500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.643 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.641+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.643 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.642+0000 7f60c2279500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.643 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.642+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.643 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.642+0000 7f60c2279500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.644 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.642+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.644 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.643+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.653+0000 7f60c2279500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:21.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.653+0000 7f60c2279500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:21.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.653+0000 7f60c2279500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:21.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.653+0000 7f60c2279500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:21.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.653+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.653+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.653+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.654+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.654+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.654+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.654+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:21.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.658 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.658 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.659 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.659 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 20 zonegroup default 2023-09-22T13:38:21.659 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.655+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.659 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.656+0000 7f60c2279500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.656+0000 7f60c2279500 10 Cannot find current period zone using local zone 2023-09-22T13:38:21.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.656+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.656+0000 7f60c2279500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.656+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.656+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.656+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.656+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.656+0000 7f60c2279500 20 zone default found 2023-09-22T13:38:21.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.656+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.657+0000 7f60c2279500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.657+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.661 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.657+0000 7f60c2279500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.661 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.657+0000 7f60c2279500 20 started sync module instance, tier type = 2023-09-22T13:38:21.661 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.657+0000 7f60c2279500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:21.669 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.667+0000 7f60c2279500 20 add_watcher() i=0 2023-09-22T13:38:21.669 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.667+0000 7f60c2279500 20 add_watcher() i=1 2023-09-22T13:38:21.669 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.667+0000 7f60c2279500 20 add_watcher() i=2 2023-09-22T13:38:21.669 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.667+0000 7f60c2279500 20 add_watcher() i=3 2023-09-22T13:38:21.669 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.667+0000 7f60c2279500 20 add_watcher() i=4 2023-09-22T13:38:21.669 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.667+0000 7f60c2279500 20 add_watcher() i=5 2023-09-22T13:38:21.669 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.667+0000 7f60c2279500 20 add_watcher() i=6 2023-09-22T13:38:21.670 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.668+0000 7f60ad75b700 0 -- 172.21.15.69:0/4292847441 >> [v2:172.21.15.69:6800/145165,v1:172.21.15.69:6801/145165] conn(0x7f60a001f7b0 msgr2=0x7f60a007a050 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.670 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.668+0000 7f60c2279500 20 add_watcher() i=7 2023-09-22T13:38:21.670 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.668+0000 7f60c2279500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:21.671 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.670+0000 7f607e7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.672 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.670+0000 7f60c2279500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:21.672 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.670+0000 7f60c2279500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:21.675 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.674+0000 7f60acf5a700 0 -- 172.21.15.69:0/3943098006 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x55e8c4fcfc90 msgr2=0x55e8c4fe19a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.677 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.676+0000 7f60c2279500 5 note: GC not initialized 2023-09-22T13:38:21.679 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.677+0000 7f5fed7fa700 20 reqs_thread_entry: start 2023-09-22T13:38:21.679 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.677+0000 7f5fe6ffd700 20 reqs_thread_entry: start 2023-09-22T13:38:21.679 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.677+0000 7f60c2279500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:21.680 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.677+0000 7f60c2279500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:21.680 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.677+0000 7f5fe57fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:51 2023 2023-09-22T13:38:21.680 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.680 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.677+0000 7f5fe4ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:21.680 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.677+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cadab0 obj=default.rgw.meta:root:check-bucket state=0x55e8c4f3bb70 s->prefetch_data=0 2023-09-22T13:38:21.680 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.678+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.680 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.678+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.681 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.678+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:21.681 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.678+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cadab0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.681 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.679+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.681 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.679+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.681 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.679+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.681 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.679+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.681 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.679+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.682 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.679+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.682 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.679+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cad880 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.683 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.680+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.683 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.680+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.683 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.680+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.683 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.680+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.683 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.680+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.683 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.680+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.683 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.680+0000 7f60c2279500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:21.683 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.681+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cad880 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.683 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.681+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.684 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.681+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.684 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.681+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.684 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.681+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.685 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.682+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.685 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.682+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.685 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.682+0000 7f60c2279500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:21.685 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.682+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cad880 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.686 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.682+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.686 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.682+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.686 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.682+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.686 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.682+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.686 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.683+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.686 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.683+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.686 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.683+0000 7f60c2279500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:21.686 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.683+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cad880 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.686 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.683+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.687 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.683+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.687 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.683+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.687 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.683+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.687 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.684+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.687 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.684+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.687 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.684+0000 7f60c2279500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:21.688 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.684+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cad880 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.688 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.684+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.688 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.684+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.689 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.684+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.689 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.684+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.689 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.685+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.689 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.685+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.689 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.685+0000 7f60c2279500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:21.689 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.685+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cad880 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.689 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.686+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.689 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.686+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.689 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.686+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.689 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.686+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.690 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.686+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.690 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.686+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.690 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.686+0000 7f60c2279500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:21.690 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.687+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cad880 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.691 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.687+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.691 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.687+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.692 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.687+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.692 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.687+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.692 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.688+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.692 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.688+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.692 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.688+0000 7f60c2279500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:21.692 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.688+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cad880 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.692 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.689+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.692 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.689+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.692 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.689+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.693 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.689+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.693 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.689+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.693 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.689+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.693 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.689+0000 7f60c2279500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:21.693 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.690+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cad880 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.693 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.690+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.693 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.690+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.693 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.690+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.693 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.690+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.695 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.690+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.695 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.690+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.695 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.690+0000 7f60c2279500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:21.695 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.691+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cad880 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.695 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.691+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.695 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.691+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.695 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.691+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.696 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.691+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.696 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.691+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.696 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.691+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.696 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.691+0000 7f60c2279500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:21.696 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.692+0000 7f60c2279500 20 get_system_obj_state: rctx=0x7ffc91cad880 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55e8c4e970c0 s->prefetch_data=0 2023-09-22T13:38:21.696 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.692+0000 7f60c2279500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.696 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.692+0000 7f60c2279500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.696 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.692+0000 7f60c2279500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.696 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.692+0000 7f60c2279500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.697 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.692+0000 7f60c2279500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.697 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.692+0000 7f60c2279500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.697 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.692+0000 7f60c2279500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:21.697 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.694+0000 7f60c2279500 20 remove_watcher() i=0 2023-09-22T13:38:21.697 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.694+0000 7f60c2279500 2 removed watcher, disabling cache 2023-09-22T13:38:21.697 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.695+0000 7f60c2279500 20 remove_watcher() i=1 2023-09-22T13:38:21.698 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.696+0000 7f60c2279500 20 remove_watcher() i=2 2023-09-22T13:38:21.699 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.697+0000 7f60c2279500 20 remove_watcher() i=3 2023-09-22T13:38:21.700 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.699+0000 7f60c2279500 20 remove_watcher() i=4 2023-09-22T13:38:21.701 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.699+0000 7f60c2279500 20 remove_watcher() i=5 2023-09-22T13:38:21.702 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.700+0000 7f60c2279500 20 remove_watcher() i=6 2023-09-22T13:38:21.703 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.701+0000 7f60c2279500 20 remove_watcher() i=7 2023-09-22T13:38:21.704 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.702+0000 7f607e7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.710 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:38:21.734 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:21.734 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:21.742 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.740+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.744 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.742+0000 7fce250ab500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.744 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.742+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.744 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.743+0000 7fce250ab500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.744 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.743+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.744 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.743+0000 7fce250ab500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.744 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.743+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.745 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.744+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.755 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.754+0000 7fce250ab500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:21.756 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.754+0000 7fce250ab500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:21.756 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.754+0000 7fce250ab500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:21.756 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.754+0000 7fce250ab500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:21.756 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.754+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.756 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.754+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.756 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.754+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.757 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.754+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.757 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.754+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.757 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.755+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.757 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.755+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.758 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.755+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.758 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.755+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.758 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.755+0000 7fce250ab500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.758 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.755+0000 7fce250ab500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:21.758 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.755+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.758 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.758 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.758 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.758 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 20 zonegroup default 2023-09-22T13:38:21.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.760 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 10 Cannot find current period zone using local zone 2023-09-22T13:38:21.760 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.760 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.760 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.756+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.760 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.757+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.760 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.757+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.760 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.757+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.760 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.757+0000 7fce250ab500 20 zone default found 2023-09-22T13:38:21.760 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.757+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.760 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.758+0000 7fce250ab500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.760 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.758+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.761 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.758+0000 7fce250ab500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.761 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.758+0000 7fce250ab500 20 started sync module instance, tier type = 2023-09-22T13:38:21.761 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.758+0000 7fce250ab500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:21.770 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.768+0000 7fce1058d700 0 -- 172.21.15.69:0/1889860120 >> [v2:172.21.15.69:6800/145165,v1:172.21.15.69:6801/145165] conn(0x7fce0404d4a0 msgr2=0x7fce0404c550 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.770 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.769+0000 7fce250ab500 20 add_watcher() i=0 2023-09-22T13:38:21.771 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.769+0000 7fce250ab500 20 add_watcher() i=1 2023-09-22T13:38:21.771 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.769+0000 7fce250ab500 20 add_watcher() i=2 2023-09-22T13:38:21.771 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.769+0000 7fce250ab500 20 add_watcher() i=3 2023-09-22T13:38:21.771 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.769+0000 7fce250ab500 20 add_watcher() i=4 2023-09-22T13:38:21.771 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.769+0000 7fce250ab500 20 add_watcher() i=5 2023-09-22T13:38:21.771 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.769+0000 7fce250ab500 20 add_watcher() i=6 2023-09-22T13:38:21.773 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.771+0000 7fce250ab500 20 add_watcher() i=7 2023-09-22T13:38:21.773 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.771+0000 7fce250ab500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:21.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.773+0000 7fcde1fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.774+0000 7fce250ab500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:21.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.774+0000 7fce250ab500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:21.779 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.778+0000 7fce0fd8c700 0 -- 172.21.15.69:0/375261665 >> [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] conn(0x55d09e19ad10 msgr2=0x55d09e1ab700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:38:21.781 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.780+0000 7fce250ab500 5 note: GC not initialized 2023-09-22T13:38:21.783 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.781+0000 7fcd58ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:21.783 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.781+0000 7fcd4e7fc700 20 reqs_thread_entry: start 2023-09-22T13:38:21.783 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.781+0000 7fce250ab500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:21.783 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.781+0000 7fce250ab500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:21.783 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.781+0000 7fcd4cff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:51 2023 2023-09-22T13:38:21.783 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.783 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.781+0000 7fcd33fff700 20 reqs_thread_entry: start 2023-09-22T13:38:21.783 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.781+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811830 obj=default.rgw.meta:root:check-bucket state=0x55d09e16cc30 s->prefetch_data=0 2023-09-22T13:38:21.784 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.782+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.784 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.782+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.782+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.782+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811830 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.782+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.782+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.782+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.782+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.783+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.783+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.783+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811600 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.783+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.783+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.785 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.783+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.786 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.783+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.786 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.783+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.783+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.783+0000 7fce250ab500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:21.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.784+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811600 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.784+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.784+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.784+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.784+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.785+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.785+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.785+0000 7fce250ab500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:21.787 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.785+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811600 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.786+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.786+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.788 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.786+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.786+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.786+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.786+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.786+0000 7fce250ab500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:21.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.786+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811600 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.787+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.787+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.787+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.787+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.787+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.787+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.787+0000 7fce250ab500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:21.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.788+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811600 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.788+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.788+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.788+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.788+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.788+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.788+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.788+0000 7fce250ab500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:21.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.789+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811600 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.789+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.789+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.789+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.789+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.789+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.789+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.789+0000 7fce250ab500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.790+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811600 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.790+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.790+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.790+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.790+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.790+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.790+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.790+0000 7fce250ab500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:21.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.791+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811600 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.791+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.791+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.791+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.791+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.792+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.792+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.792+0000 7fce250ab500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:21.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.792+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811600 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.795 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.792+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.795 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.792+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.795 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.792+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.795 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.792+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.795 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.793+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.796 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.793+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.796 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.793+0000 7fce250ab500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:21.796 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.793+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811600 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.796 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.794+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.796 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.794+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.796 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.794+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.796 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.794+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.797 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.794+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.797 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.794+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.797 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.794+0000 7fce250ab500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:21.797 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.794+0000 7fce250ab500 20 get_system_obj_state: rctx=0x7ffc9d811600 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d09e16c530 s->prefetch_data=0 2023-09-22T13:38:21.797 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.795+0000 7fce250ab500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:21.797 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.795+0000 7fce250ab500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:21.797 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.795+0000 7fce250ab500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:21.797 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.795+0000 7fce250ab500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.797 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.795+0000 7fce250ab500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:21.797 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.795+0000 7fce250ab500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:21.798 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.795+0000 7fce250ab500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:21.798 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.797+0000 7fce250ab500 20 remove_watcher() i=0 2023-09-22T13:38:21.799 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.797+0000 7fce250ab500 2 removed watcher, disabling cache 2023-09-22T13:38:21.800 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.798+0000 7fce250ab500 20 remove_watcher() i=1 2023-09-22T13:38:21.801 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.799+0000 7fce250ab500 20 remove_watcher() i=2 2023-09-22T13:38:21.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.800+0000 7fce250ab500 20 remove_watcher() i=3 2023-09-22T13:38:21.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.801+0000 7fce250ab500 20 remove_watcher() i=4 2023-09-22T13:38:21.804 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.803+0000 7fce250ab500 20 remove_watcher() i=5 2023-09-22T13:38:21.805 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.803+0000 7fce250ab500 20 remove_watcher() i=6 2023-09-22T13:38:21.806 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.804+0000 7fce250ab500 20 remove_watcher() i=7 2023-09-22T13:38:21.806 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.805+0000 7fcde1fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.811 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: bucket check olh does not find new leftover OLH entries after fix 2023-09-22T13:38:21.811 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:21.813 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bucket check olh --bucket check-bucket --dump-keys 2023-09-22T13:38:21.837 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:21.838 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:21.845 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.844+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.847 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.845+0000 7f39b8e51500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.847 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.845+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.847 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.846+0000 7f39b8e51500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.847 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.846+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.848 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.846+0000 7f39b8e51500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.848 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.846+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.848 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.847+0000 7f39b8e51500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.858+0000 7f39b8e51500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:21.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.858+0000 7f39b8e51500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:21.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.858+0000 7f39b8e51500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:21.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.858+0000 7f39b8e51500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:21.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.858+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.858+0000 7f39b8e51500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.860 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.858+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.861 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.859+0000 7f39b8e51500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.861 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.859+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.861 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.859+0000 7f39b8e51500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.861 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.859+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.859+0000 7f39b8e51500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.859+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.860+0000 7f39b8e51500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.860+0000 7f39b8e51500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:21.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.860+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.860+0000 7f39b8e51500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.862 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.860+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.863 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.860+0000 7f39b8e51500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.860+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.860+0000 7f39b8e51500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:21.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.860+0000 7f39b8e51500 20 zonegroup default 2023-09-22T13:38:21.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.860+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.861+0000 7f39b8e51500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.861+0000 7f39b8e51500 10 Cannot find current period zone using local zone 2023-09-22T13:38:21.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.861+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.861+0000 7f39b8e51500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.861+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.861+0000 7f39b8e51500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:21.864 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.861+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.865 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.861+0000 7f39b8e51500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:21.865 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.861+0000 7f39b8e51500 20 zone default found 2023-09-22T13:38:21.865 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.861+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.865 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.862+0000 7f39b8e51500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.865 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.862+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:21.865 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.862+0000 7f39b8e51500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:21.865 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.862+0000 7f39b8e51500 20 started sync module instance, tier type = 2023-09-22T13:38:21.865 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.862+0000 7f39b8e51500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:21.879 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.878+0000 7f39b8e51500 20 add_watcher() i=0 2023-09-22T13:38:21.880 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.878+0000 7f39b8e51500 20 add_watcher() i=1 2023-09-22T13:38:21.880 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.878+0000 7f39b8e51500 20 add_watcher() i=2 2023-09-22T13:38:21.880 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.878+0000 7f39b8e51500 20 add_watcher() i=3 2023-09-22T13:38:21.880 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.878+0000 7f39b8e51500 20 add_watcher() i=4 2023-09-22T13:38:21.880 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.878+0000 7f39b8e51500 20 add_watcher() i=5 2023-09-22T13:38:21.882 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.880+0000 7f39a4333700 0 -- 172.21.15.69:0/4172442731 >> [v2:172.21.15.69:6808/145166,v1:172.21.15.69:6809/145166] conn(0x55d5271f2650 msgr2=0x55d5271f8770 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:21.882 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.881+0000 7f39b8e51500 20 add_watcher() i=6 2023-09-22T13:38:21.882 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.881+0000 7f39b8e51500 20 add_watcher() i=7 2023-09-22T13:38:21.882 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.881+0000 7f39b8e51500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:21.885 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.883+0000 7f3975fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:21.885 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.884+0000 7f39b8e51500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:21.885 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.884+0000 7f39b8e51500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:21.889 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:21.888+0000 7f39a4333700 0 -- 172.21.15.69:0/3396389141 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x55d52723aa90 msgr2=0x55d52724cd80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:22.090 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.089+0000 7f39a4333700 0 --2- 172.21.15.69:0/3396389141 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x55d52723aa90 0x55d52724cd80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:38:22.092 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.091+0000 7f39b8e51500 5 note: GC not initialized 2023-09-22T13:38:22.094 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.092+0000 7f38ecff9700 20 reqs_thread_entry: start 2023-09-22T13:38:22.094 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.092+0000 7f38e27fc700 20 reqs_thread_entry: start 2023-09-22T13:38:22.094 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.092+0000 7f39b8e51500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:22.095 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:22.095 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f38e0ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:52 2023 2023-09-22T13:38:22.095 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:22.095 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f38c7fff700 20 reqs_thread_entry: start 2023-09-22T13:38:22.095 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 20 get_system_obj_state: rctx=0x7ffc15de6f58 obj=default.rgw.meta:root:check-bucket state=0x55d527240cc0 s->prefetch_data=0 2023-09-22T13:38:22.095 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:38:22.097 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:38:22.097 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:22.097 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:22.097 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:22.097 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:38:22.097 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:22.097 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:22.097 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:38:22.097 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:22.098 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 20 get_system_obj_state: rctx=0x7ffc15de7028 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55d527240cc0 s->prefetch_data=0 2023-09-22T13:38:22.098 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.093+0000 7f39b8e51500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : miss 2023-09-22T13:38:22.098 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.094+0000 7f39b8e51500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x16 2023-09-22T13:38:22.098 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.094+0000 7f39b8e51500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:22.098 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.094+0000 7f39b8e51500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:22.098 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.094+0000 7f39b8e51500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:22.098 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.094+0000 7f39b8e51500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:22.098 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.094+0000 7f39b8e51500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:22.098 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.094+0000 7f39b8e51500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:22.098 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.094+0000 7f39b8e51500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:38:22.098 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.094+0000 7f39b8e51500 20 rados->read ofs=0 len=0 2023-09-22T13:38:22.099 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.095+0000 7f39b8e51500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:22.099 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.095+0000 7f39b8e51500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x13 2023-09-22T13:38:22.099 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.095+0000 7f39b8e51500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:22.099 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.095+0000 7f39b8e51500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:22.099 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.095+0000 7f39b8e51500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:22.099 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.095+0000 7f39b8e51500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 2023-09-22T13:38:22.099 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.095+0000 7f39b8e51500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:22.099 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.095+0000 7f39b8e51500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:38:22.099 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.095+0000 7f39b8e51500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:22.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.096+0000 7f39b8e51500 1 NOTICE: finished shard 1 (0 entries found) 2023-09-22T13:38:22.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.096+0000 7f39b8e51500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:22.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.096+0000 7f39b8e51500 1 NOTICE: finished shard 2 (0 entries found) 2023-09-22T13:38:22.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.096+0000 7f39b8e51500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:22.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.097+0000 7f39b8e51500 1 NOTICE: finished shard 3 (0 entries found) 2023-09-22T13:38:22.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.097+0000 7f39b8e51500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:22.101 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.097+0000 7f39b8e51500 1 NOTICE: finished shard 4 (0 entries found) 2023-09-22T13:38:22.102 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.097+0000 7f39b8e51500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:22.102 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.098+0000 7f39b8e51500 1 NOTICE: finished shard 5 (0 entries found) 2023-09-22T13:38:22.102 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.098+0000 7f39b8e51500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:22.102 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.098+0000 7f39b8e51500 1 NOTICE: finished shard 6 (0 entries found) 2023-09-22T13:38:22.102 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.098+0000 7f39b8e51500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:22.102 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.099+0000 7f39b8e51500 1 NOTICE: finished shard 7 (0 entries found) 2023-09-22T13:38:22.102 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.099+0000 7f39b8e51500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:22.102 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.099+0000 7f39b8e51500 1 NOTICE: finished shard 8 (0 entries found) 2023-09-22T13:38:22.102 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.099+0000 7f39b8e51500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:22.102 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.099+0000 7f39b8e51500 1 NOTICE: finished shard 9 (0 entries found) 2023-09-22T13:38:22.102 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.099+0000 7f39b8e51500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:22.103 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.100+0000 7f39b8e51500 1 NOTICE: finished shard 10 (0 entries found) 2023-09-22T13:38:22.103 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.100+0000 7f39b8e51500 1 NOTICE: finished all shards (0 entries found) 2023-09-22T13:38:22.103 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.101+0000 7f39b8e51500 20 remove_watcher() i=0 2023-09-22T13:38:22.103 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.101+0000 7f39b8e51500 2 removed watcher, disabling cache 2023-09-22T13:38:22.104 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.103+0000 7f39b8e51500 20 remove_watcher() i=1 2023-09-22T13:38:22.107 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.105+0000 7f39b8e51500 20 remove_watcher() i=2 2023-09-22T13:38:22.108 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.107+0000 7f39b8e51500 20 remove_watcher() i=3 2023-09-22T13:38:22.109 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.108+0000 7f39b8e51500 20 remove_watcher() i=4 2023-09-22T13:38:22.113 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.112+0000 7f39b8e51500 20 remove_watcher() i=5 2023-09-22T13:38:22.114 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.113+0000 7f39b8e51500 20 remove_watcher() i=6 2023-09-22T13:38:22.115 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.114+0000 7f39b8e51500 20 remove_watcher() i=7 2023-09-22T13:38:22.116 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:22.114+0000 7f3975fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:22.121 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: verify that bucket check fixes do not affect null version objects 2023-09-22T13:38:22.121 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:22.172 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.172 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.172 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.172 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.172 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.172 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.173 INFO:tasks.workunit.client.0.smithi069.stderr:TEST: bucket check stats are correct in the presence of unlinked entries 2023-09-22T13:38:22.173 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:22.418 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.418 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.418 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.419 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.419 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.419 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.419 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.419 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.419 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.419 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.419 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.419 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.419 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.419 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.420 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.420 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.420 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.420 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.420 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.420 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.420 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.420 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.420 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.420 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.421 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.421 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.421 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:22.421 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:22.421 INFO:tasks.workunit.client.0.smithi069.stderr:ceph config set client rgw_debug_inject_set_olh_err 2 2023-09-22T13:38:22.823 INFO:tasks.workunit.client.0.smithi069.stderr:ceph config set client rgw_debug_inject_olh_cancel_modification_err true 2023-09-22T13:38:24.355 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:24.355 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:24.355 INFO:tasks.workunit.client.0.smithi069.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:38:24.357 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:38:24.385 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:24.385 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:24.394 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.393+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.396 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.395+0000 7f7090c6e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.397 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.395+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.397 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.396+0000 7f7090c6e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.397 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.396+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.397 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.396+0000 7f7090c6e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.398 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.396+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.398 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.397+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:24.411 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.410+0000 7f7090c6e500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:24.412 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.410+0000 7f7090c6e500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:24.412 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.410+0000 7f7090c6e500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:24.412 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.410+0000 7f7090c6e500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:24.412 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.410+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.412 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.410+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:24.412 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.410+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.413 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.411+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:24.413 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.411+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.413 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.411+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:24.414 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.411+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.414 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.412+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:24.414 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.412+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.414 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.412+0000 7f7090c6e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.414 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.412+0000 7f7090c6e500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:24.414 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.412+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.414 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.412+0000 7f7090c6e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.414 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.412+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.415 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.413+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:24.415 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.413+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.415 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.413+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:24.416 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.413+0000 7f7090c6e500 20 zonegroup default 2023-09-22T13:38:24.416 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.413+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.416 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.413+0000 7f7090c6e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.416 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.413+0000 7f7090c6e500 10 Cannot find current period zone using local zone 2023-09-22T13:38:24.416 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.413+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.416 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.413+0000 7f7090c6e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.416 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.413+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.416 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.414+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:24.416 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.414+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.414+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:24.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.414+0000 7f7090c6e500 20 zone default found 2023-09-22T13:38:24.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.414+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.415+0000 7f7090c6e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.415+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.415+0000 7f7090c6e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.415+0000 7f7090c6e500 20 started sync module instance, tier type = 2023-09-22T13:38:24.418 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.415+0000 7f7090c6e500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:24.423 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.422+0000 7f707c150700 0 -- 172.21.15.69:0/2946947746 >> [v2:172.21.15.69:6824/145182,v1:172.21.15.69:6825/145182] conn(0x7f70640035a0 msgr2=0x7f7074005970 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:24.434 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.433+0000 7f7090c6e500 20 add_watcher() i=0 2023-09-22T13:38:24.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.433+0000 7f7090c6e500 20 add_watcher() i=1 2023-09-22T13:38:24.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.433+0000 7f7090c6e500 20 add_watcher() i=2 2023-09-22T13:38:24.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.433+0000 7f7090c6e500 20 add_watcher() i=3 2023-09-22T13:38:24.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.433+0000 7f7090c6e500 20 add_watcher() i=4 2023-09-22T13:38:24.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.433+0000 7f7090c6e500 20 add_watcher() i=5 2023-09-22T13:38:24.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.433+0000 7f7090c6e500 20 add_watcher() i=6 2023-09-22T13:38:24.437 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.435+0000 7f7090c6e500 20 add_watcher() i=7 2023-09-22T13:38:24.437 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.435+0000 7f7090c6e500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:24.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.438+0000 7f704dfe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:24.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.439+0000 7f7090c6e500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:24.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.439+0000 7f7090c6e500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:24.642 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.641+0000 7f707b94f700 0 --2- 172.21.15.69:0/3858165690 >> [v2:172.21.15.69:3301/0,v1:172.21.15.69:6790/0] conn(0x55ad6e3454d0 0x55ad6e345910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:38:24.642 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.641+0000 7f707c951700 0 -- 172.21.15.69:0/3858165690 >> [v2:172.21.15.69:3301/0,v1:172.21.15.69:6790/0] conn(0x7f6fbc001d90 msgr2=0x7f6fbc0041e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0)._try_send injecting socket failure 2023-09-22T13:38:24.647 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.646+0000 7f7090c6e500 5 note: GC not initialized 2023-09-22T13:38:24.649 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.647+0000 7f6fc4ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:24.649 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.647+0000 7f6fba7fc700 20 reqs_thread_entry: start 2023-09-22T13:38:24.649 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.647+0000 7f7090c6e500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:24.649 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.647+0000 7f7090c6e500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:24.649 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.647+0000 7f6fb8ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:54 2023 2023-09-22T13:38:24.649 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:24.649 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.647+0000 7f6f9ffff700 20 reqs_thread_entry: start 2023-09-22T13:38:24.649 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.647+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f4b00 obj=default.rgw.meta:root:check-bucket state=0x55ad6e2180c0 s->prefetch_data=0 2023-09-22T13:38:24.650 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.648+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.650 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.648+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.650 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.648+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:24.650 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.648+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f4b00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.650 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.649+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.651 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.649+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.651 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.649+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.651 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.649+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.651 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.649+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.651 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.649+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.652 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.649+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f48d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.652 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.649+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.652 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.649+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.652 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.649+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.652 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.649+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.652 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.650+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.652 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.650+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.652 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.650+0000 7f7090c6e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:24.653 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.651+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f48d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.653 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.651+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.653 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.651+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.653 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.651+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.653 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.651+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.654 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.652+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.654 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.652+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.654 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.652+0000 7f7090c6e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:24.654 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.652+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f48d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.653+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.653+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.653+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.653+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.653+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.653+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.655 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.653+0000 7f7090c6e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:24.656 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.653+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f48d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.656 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.654+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.656 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.654+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.656 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.654+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.656 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.654+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.656 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.654+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.654+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.654+0000 7f7090c6e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:24.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.655+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f48d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.656+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.656+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.656+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.657 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.656+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.658 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.656+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.658 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.656+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.658 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.656+0000 7f7090c6e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:24.658 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.657+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f48d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.659 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.657+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.659 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.657+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.659 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.657+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.659 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.657+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.659 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.658+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.658+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.658+0000 7f7090c6e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:24.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.658+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f48d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.659+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.659+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.660 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.659+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.661 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.659+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.661 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.659+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.661 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.659+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.661 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.659+0000 7f7090c6e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:24.661 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.660+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f48d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.662 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.660+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.662 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.660+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.662 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.660+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.662 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.660+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.663 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.661+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.663 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.661+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.663 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.661+0000 7f7090c6e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:24.663 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.661+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f48d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.663 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.661+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.663 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.661+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.663 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.661+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.664 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.661+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.664 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.662+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.664 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.662+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.664 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.662+0000 7f7090c6e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:24.664 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.663+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f48d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.665 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.663+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.665 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.663+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.665 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.663+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.665 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.663+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.666 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.664+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.666 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.664+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.666 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.664+0000 7f7090c6e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:24.666 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.664+0000 7f7090c6e500 20 get_system_obj_state: rctx=0x7fff878f48d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55ad6e19a810 s->prefetch_data=0 2023-09-22T13:38:24.667 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.665+0000 7f7090c6e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:24.667 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.665+0000 7f7090c6e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:24.667 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.665+0000 7f7090c6e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:24.667 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.665+0000 7f7090c6e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.667 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.665+0000 7f7090c6e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:24.667 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.665+0000 7f7090c6e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:24.667 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.665+0000 7f7090c6e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:24.669 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.667+0000 7f7090c6e500 20 remove_watcher() i=0 2023-09-22T13:38:24.669 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.667+0000 7f7090c6e500 2 removed watcher, disabling cache 2023-09-22T13:38:24.670 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.669+0000 7f7090c6e500 20 remove_watcher() i=1 2023-09-22T13:38:24.673 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.671+0000 7f7090c6e500 20 remove_watcher() i=2 2023-09-22T13:38:24.674 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.672+0000 7f7090c6e500 20 remove_watcher() i=3 2023-09-22T13:38:24.675 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.674+0000 7f7090c6e500 20 remove_watcher() i=4 2023-09-22T13:38:24.676 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.675+0000 7f7090c6e500 20 remove_watcher() i=5 2023-09-22T13:38:24.677 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.676+0000 7f7090c6e500 20 remove_watcher() i=6 2023-09-22T13:38:24.678 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.677+0000 7f7090c6e500 20 remove_watcher() i=7 2023-09-22T13:38:24.679 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.678+0000 7f704dfe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:24.742 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:24.742 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:24.742 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:24.742 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:24.742 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:24.742 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:24.742 INFO:tasks.workunit.client.0.smithi069.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:38:24.743 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:38:24.767 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:24.767 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:24.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.773+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.777 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.775+0000 7f1ec0308500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.777 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.775+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.777 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.776+0000 7f1ec0308500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.778 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.776+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.778 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.776+0000 7f1ec0308500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.778 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.776+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.778 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.777+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:24.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.787+0000 7f1ec0308500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:24.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.787+0000 7f1ec0308500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:24.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.787+0000 7f1ec0308500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:24.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.787+0000 7f1ec0308500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:24.789 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.787+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.788+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:24.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.788+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.788+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:24.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.788+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.788+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.788+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.788+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.788+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.789+0000 7f1ec0308500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.789+0000 7f1ec0308500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.789+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.789+0000 7f1ec0308500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.789+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.789+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.789+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.792 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.789+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:24.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.789+0000 7f1ec0308500 20 zonegroup default 2023-09-22T13:38:24.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.789+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.790+0000 7f1ec0308500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.790+0000 7f1ec0308500 10 Cannot find current period zone using local zone 2023-09-22T13:38:24.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.790+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.790+0000 7f1ec0308500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.790+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.790+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:24.793 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.790+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.790+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:24.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.790+0000 7f1ec0308500 20 zone default found 2023-09-22T13:38:24.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.790+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.794 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.791+0000 7f1ec0308500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.795 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.791+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:24.795 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.791+0000 7f1ec0308500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:24.795 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.791+0000 7f1ec0308500 20 started sync module instance, tier type = 2023-09-22T13:38:24.795 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.791+0000 7f1ec0308500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:24.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.800+0000 7f1ec0308500 20 add_watcher() i=0 2023-09-22T13:38:24.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.800+0000 7f1ec0308500 20 add_watcher() i=1 2023-09-22T13:38:24.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.800+0000 7f1ec0308500 20 add_watcher() i=2 2023-09-22T13:38:24.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.800+0000 7f1ec0308500 20 add_watcher() i=3 2023-09-22T13:38:24.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.800+0000 7f1ec0308500 20 add_watcher() i=4 2023-09-22T13:38:24.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.800+0000 7f1ec0308500 20 add_watcher() i=5 2023-09-22T13:38:24.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.800+0000 7f1ec0308500 20 add_watcher() i=6 2023-09-22T13:38:24.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.801+0000 7f1eab7ea700 0 -- 172.21.15.69:0/1376870929 >> [v2:172.21.15.69:6800/145165,v1:172.21.15.69:6801/145165] conn(0x7f1ea00073f0 msgr2=0x7f1ea0007830 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:24.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.801+0000 7f1ec0308500 20 add_watcher() i=7 2023-09-22T13:38:24.803 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.801+0000 7f1ec0308500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:24.804 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.803+0000 7f1df7fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:24.805 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.803+0000 7f1ec0308500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:24.805 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.803+0000 7f1ec0308500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:24.809 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:24.808+0000 7f1eabfeb700 0 -- 172.21.15.69:0/17725477 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x557c084da6a0 msgr2=0x557c084fc830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:25.010 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.009+0000 7f1eabfeb700 0 --2- 172.21.15.69:0/17725477 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x557c084da6a0 0x557c084fc830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:38:25.012 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.011+0000 7f1ec0308500 5 note: GC not initialized 2023-09-22T13:38:25.013 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.012+0000 7f1de6ffd700 20 reqs_thread_entry: start 2023-09-22T13:38:25.014 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.012+0000 7f1de4ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:25.014 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.012+0000 7f1ec0308500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:25.014 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.012+0000 7f1ec0308500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:25.014 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.012+0000 7f1ddeffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:55 2023 2023-09-22T13:38:25.014 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:25.014 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.012+0000 7f1dde7fc700 20 reqs_thread_entry: start 2023-09-22T13:38:25.014 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.012+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e260 obj=default.rgw.meta:root:check-bucket state=0x557c084de020 s->prefetch_data=0 2023-09-22T13:38:25.015 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.012+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.015 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.012+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.015 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.013+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:25.015 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.013+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e260 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.015 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.013+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.015 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.013+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.015 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.013+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.013+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.014+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.014+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.014+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e030 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.014+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.014+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.014+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.017 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.014+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.017 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.015+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.017 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.015+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.017 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.015+0000 7f1ec0308500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:25.017 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.016+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e030 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.018 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.016+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.018 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.016+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.018 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.016+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.018 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.017+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.019 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.017+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.019 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.017+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.019 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.017+0000 7f1ec0308500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:25.019 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.018+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e030 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.020 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.018+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.020 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.018+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.020 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.018+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.020 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.018+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.020 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.018+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.020 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.018+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.021 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.018+0000 7f1ec0308500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:25.021 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.019+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e030 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.021 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.019+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.021 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.019+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.021 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.019+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.022 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.019+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.022 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.020+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.022 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.020+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.022 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.020+0000 7f1ec0308500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:25.022 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.020+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e030 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.023 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.021+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.023 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.021+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.023 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.021+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.023 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.021+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.023 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.021+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.023 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.021+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.023 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.021+0000 7f1ec0308500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:25.024 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.022+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e030 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.024 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.022+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.024 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.022+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.024 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.022+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.024 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.022+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.024 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.023+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.024 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.023+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.024 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.023+0000 7f1ec0308500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:25.025 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.023+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e030 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.025 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.024+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.025 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.024+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.026 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.024+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.026 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.024+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.026 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.024+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.026 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.024+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.026 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.024+0000 7f1ec0308500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:25.027 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.025+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e030 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.027 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.025+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.027 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.025+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.027 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.025+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.027 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.025+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.028 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.026+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.028 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.026+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.028 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.026+0000 7f1ec0308500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:25.028 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.026+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e030 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.028 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.027+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.029 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.027+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.029 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.027+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.029 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.027+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.029 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.027+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.029 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.027+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.029 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.027+0000 7f1ec0308500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:25.030 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.028+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e030 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.030 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.028+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.030 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.028+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.030 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.028+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.030 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.028+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.030 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.029+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.031 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.029+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.031 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.029+0000 7f1ec0308500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:25.031 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.029+0000 7f1ec0308500 20 get_system_obj_state: rctx=0x7ffe6062e030 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x557c08506290 s->prefetch_data=0 2023-09-22T13:38:25.031 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.029+0000 7f1ec0308500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.031 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.029+0000 7f1ec0308500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.032 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.029+0000 7f1ec0308500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.032 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.029+0000 7f1ec0308500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.032 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.030+0000 7f1ec0308500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.032 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.030+0000 7f1ec0308500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.032 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.030+0000 7f1ec0308500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:25.033 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.032+0000 7f1ec0308500 20 remove_watcher() i=0 2023-09-22T13:38:25.033 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.032+0000 7f1ec0308500 2 removed watcher, disabling cache 2023-09-22T13:38:25.034 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.033+0000 7f1ec0308500 20 remove_watcher() i=1 2023-09-22T13:38:25.036 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.034+0000 7f1ec0308500 20 remove_watcher() i=2 2023-09-22T13:38:25.037 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.036+0000 7f1ec0308500 20 remove_watcher() i=3 2023-09-22T13:38:25.038 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.037+0000 7f1ec0308500 20 remove_watcher() i=4 2023-09-22T13:38:25.039 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.038+0000 7f1ec0308500 20 remove_watcher() i=5 2023-09-22T13:38:25.040 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.039+0000 7f1ec0308500 20 remove_watcher() i=6 2023-09-22T13:38:25.041 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.040+0000 7f1ec0308500 20 remove_watcher() i=7 2023-09-22T13:38:25.042 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.040+0000 7f1df7fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:25.107 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:25.107 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:25.107 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:25.107 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:25.107 INFO:tasks.workunit.client.0.smithi069.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:38:25.108 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object e 2023-09-22T13:38:25.132 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:25.132 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:25.140 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.138+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.141 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.139+0000 7f4d9ff4a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.141 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.139+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.142 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.140+0000 7f4d9ff4a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.142 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.140+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.142 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.140+0000 7f4d9ff4a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.142 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.140+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.143 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.141+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:25.154 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.153+0000 7f4d9ff4a500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:25.154 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.153+0000 7f4d9ff4a500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:25.154 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.153+0000 7f4d9ff4a500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:25.154 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.153+0000 7f4d9ff4a500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:25.155 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.153+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.155 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.153+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:25.156 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.153+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.156 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.153+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:25.156 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.153+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.156 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.154+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:25.156 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.154+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.156 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.154+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:25.156 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.154+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.156 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.154+0000 7f4d9ff4a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.156 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.154+0000 7f4d9ff4a500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:25.156 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.154+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.157 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.154+0000 7f4d9ff4a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.157 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.154+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.157 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.155+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:25.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.155+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.155+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:25.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.155+0000 7f4d9ff4a500 20 zonegroup default 2023-09-22T13:38:25.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.155+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.155+0000 7f4d9ff4a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.155+0000 7f4d9ff4a500 10 Cannot find current period zone using local zone 2023-09-22T13:38:25.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.155+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.155+0000 7f4d9ff4a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.155+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.156+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:25.158 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.156+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.161 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.156+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:25.161 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.156+0000 7f4d9ff4a500 20 zone default found 2023-09-22T13:38:25.162 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.156+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.162 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.156+0000 7f4d9ff4a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.162 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.156+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.162 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.157+0000 7f4d9ff4a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.162 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.157+0000 7f4d9ff4a500 20 started sync module instance, tier type = 2023-09-22T13:38:25.162 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.157+0000 7f4d9ff4a500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:25.170 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.169+0000 7f4d9ff4a500 20 add_watcher() i=0 2023-09-22T13:38:25.170 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.169+0000 7f4d9ff4a500 20 add_watcher() i=1 2023-09-22T13:38:25.171 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.169+0000 7f4d9ff4a500 20 add_watcher() i=2 2023-09-22T13:38:25.171 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.169+0000 7f4d9ff4a500 20 add_watcher() i=3 2023-09-22T13:38:25.171 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.169+0000 7f4d9ff4a500 20 add_watcher() i=4 2023-09-22T13:38:25.171 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.169+0000 7f4d9ff4a500 20 add_watcher() i=5 2023-09-22T13:38:25.171 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.169+0000 7f4d9ff4a500 20 add_watcher() i=6 2023-09-22T13:38:25.171 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.170+0000 7f4d9ff4a500 20 add_watcher() i=7 2023-09-22T13:38:25.172 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.170+0000 7f4d9ff4a500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:25.174 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.173+0000 7f4cd7fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:25.175 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.173+0000 7f4d9ff4a500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:25.175 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.173+0000 7f4d9ff4a500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:25.176 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.174+0000 7f4d8ac2b700 0 -- 172.21.15.69:0/1863148484 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x562b01bec070 msgr2=0x562b01c01390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2023-09-22T13:38:25.181 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.179+0000 7f4d9ff4a500 5 note: GC not initialized 2023-09-22T13:38:25.183 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.180+0000 7f4cceffd700 20 reqs_thread_entry: start 2023-09-22T13:38:25.183 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.180+0000 7f4cccff9700 20 reqs_thread_entry: start 2023-09-22T13:38:25.183 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.180+0000 7f4d9ff4a500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:25.183 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.181+0000 7f4d9ff4a500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:25.183 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.181+0000 7f4cbaffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:55 2023 2023-09-22T13:38:25.183 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:25.183 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.181+0000 7f4cba7fc700 20 reqs_thread_entry: start 2023-09-22T13:38:25.183 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.181+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb990 obj=default.rgw.meta:root:check-bucket state=0x562b01bf6480 s->prefetch_data=0 2023-09-22T13:38:25.184 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.181+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.184 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.181+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.184 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.181+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:25.184 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.181+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb990 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.185 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.182+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.185 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.182+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.185 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.182+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.185 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.182+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.185 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.182+0000 7f4d8bc2d700 0 -- 172.21.15.69:0/3362723907 >> [v2:172.21.15.174:6816/145171,v1:172.21.15.174:6817/145171] conn(0x7f4d8004d4d0 msgr2=0x7f4d80079040 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:25.185 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.183+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.185 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.183+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.185 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.183+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb760 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.186 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.183+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.186 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.183+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.186 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.183+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.186 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.183+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.186 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.184+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.186 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.184+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.187 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.184+0000 7f4d9ff4a500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:25.187 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.184+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb760 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.187 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.185+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.188 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.185+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.188 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.185+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.188 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.185+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.188 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.185+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.188 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.185+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.188 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.185+0000 7f4d9ff4a500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:25.188 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.186+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb760 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.188 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.186+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.188 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.186+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.188 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.186+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.188 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.186+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.190 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.186+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.190 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.186+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.190 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.186+0000 7f4d9ff4a500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:25.190 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.187+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb760 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.190 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.187+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.190 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.187+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.190 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.187+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.190 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.187+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.190 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.188+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.191 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.188+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.191 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.188+0000 7f4d9ff4a500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:25.191 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.188+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb760 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.191 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.188+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.191 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.188+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.191 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.188+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.191 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.188+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.192 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.189+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.192 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.189+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.192 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.189+0000 7f4d9ff4a500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:25.192 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.189+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb760 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.192 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.190+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.192 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.190+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.193 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.190+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.193 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.190+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.193 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.190+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.193 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.190+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.193 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.190+0000 7f4d9ff4a500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:25.193 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.191+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb760 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.193 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.191+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.194 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.191+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.194 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.191+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.194 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.191+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.194 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.192+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.194 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.192+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.194 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.192+0000 7f4d9ff4a500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:25.194 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.192+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb760 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.195 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.192+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.195 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.192+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.195 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.192+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.192+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.193+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.193+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.193+0000 7f4d9ff4a500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.193+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb760 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.193+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.193+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.193+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.193+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.194+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.194+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.196 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.194+0000 7f4d9ff4a500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:25.197 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.194+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb760 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.197 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.195+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.197 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.195+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.197 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.195+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.198 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.195+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.198 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.195+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.198 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.195+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.198 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.195+0000 7f4d9ff4a500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:25.198 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.196+0000 7f4d9ff4a500 20 get_system_obj_state: rctx=0x7ffcc59eb760 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x562b01bf7920 s->prefetch_data=0 2023-09-22T13:38:25.199 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.196+0000 7f4d9ff4a500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.199 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.196+0000 7f4d9ff4a500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.199 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.196+0000 7f4d9ff4a500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.199 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.196+0000 7f4d9ff4a500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.199 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.197+0000 7f4d9ff4a500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.199 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.197+0000 7f4d9ff4a500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.199 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.197+0000 7f4d9ff4a500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:25.200 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.199+0000 7f4d9ff4a500 20 remove_watcher() i=0 2023-09-22T13:38:25.200 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.199+0000 7f4d9ff4a500 2 removed watcher, disabling cache 2023-09-22T13:38:25.201 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.200+0000 7f4d9ff4a500 20 remove_watcher() i=1 2023-09-22T13:38:25.204 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.203+0000 7f4d9ff4a500 20 remove_watcher() i=2 2023-09-22T13:38:25.205 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.204+0000 7f4d9ff4a500 20 remove_watcher() i=3 2023-09-22T13:38:25.206 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.205+0000 7f4d9ff4a500 20 remove_watcher() i=4 2023-09-22T13:38:25.207 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.206+0000 7f4d9ff4a500 20 remove_watcher() i=5 2023-09-22T13:38:25.209 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.208+0000 7f4d9ff4a500 20 remove_watcher() i=6 2023-09-22T13:38:25.210 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.209+0000 7f4d9ff4a500 20 remove_watcher() i=7 2023-09-22T13:38:25.211 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.209+0000 7f4cd7fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:25.269 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:25.270 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:25.270 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:25.270 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:25.270 INFO:tasks.workunit.client.0.smithi069.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:38:25.271 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bi list --bucket check-bucket --object f 2023-09-22T13:38:25.295 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:25.295 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:25.303 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.302+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.305 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.303+0000 7fc13fe4e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.305 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.303+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.305 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.304+0000 7fc13fe4e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.305 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.304+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.306 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.304+0000 7fc13fe4e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.306 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.304+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.306 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.305+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:25.317 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.315+0000 7fc13fe4e500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:25.317 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.315+0000 7fc13fe4e500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:25.317 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.315+0000 7fc13fe4e500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:25.317 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.315+0000 7fc13fe4e500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:25.317 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.315+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.318 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.316+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:25.318 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.316+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.318 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.316+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:25.318 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.316+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.319 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.317+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:25.319 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.317+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.319 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.317+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:25.319 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.317+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.319 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.317+0000 7fc13fe4e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.319 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.317+0000 7fc13fe4e500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:25.319 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.317+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.319 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.317+0000 7fc13fe4e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.319 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.317+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.320 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.318+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:25.320 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.318+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.320 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.318+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:25.320 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.318+0000 7fc13fe4e500 20 zonegroup default 2023-09-22T13:38:25.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.318+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.318+0000 7fc13fe4e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.318+0000 7fc13fe4e500 10 Cannot find current period zone using local zone 2023-09-22T13:38:25.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.318+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.319+0000 7fc13fe4e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.319+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.319+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:25.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.319+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.319+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:25.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.319+0000 7fc13fe4e500 20 zone default found 2023-09-22T13:38:25.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.319+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.320+0000 7fc13fe4e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.320+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.320+0000 7fc13fe4e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:25.322 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.320+0000 7fc13fe4e500 20 started sync module instance, tier type = 2023-09-22T13:38:25.323 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.320+0000 7fc13fe4e500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:25.338 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.337+0000 7fc13fe4e500 20 add_watcher() i=0 2023-09-22T13:38:25.338 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.337+0000 7fc13fe4e500 20 add_watcher() i=1 2023-09-22T13:38:25.339 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.337+0000 7fc13fe4e500 20 add_watcher() i=2 2023-09-22T13:38:25.339 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.337+0000 7fc13fe4e500 20 add_watcher() i=3 2023-09-22T13:38:25.339 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.337+0000 7fc13fe4e500 20 add_watcher() i=4 2023-09-22T13:38:25.339 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.337+0000 7fc13fe4e500 20 add_watcher() i=5 2023-09-22T13:38:25.339 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.337+0000 7fc13fe4e500 20 add_watcher() i=6 2023-09-22T13:38:25.339 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.338+0000 7fc13fe4e500 20 add_watcher() i=7 2023-09-22T13:38:25.339 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.338+0000 7fc13fe4e500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:25.341 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.339+0000 7fc0fcfe1700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:25.341 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.340+0000 7fc13fe4e500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:25.342 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.340+0000 7fc13fe4e500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:25.342 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.341+0000 7fc12bb31700 0 -- >> [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] conn(0x55f7da804690 msgr2=0x55f7da804a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2023-09-22T13:38:25.347 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.345+0000 7fc13fe4e500 5 note: GC not initialized 2023-09-22T13:38:25.348 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.346+0000 7fc06b7fe700 20 reqs_thread_entry: start 2023-09-22T13:38:25.348 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.347+0000 7fc0697fa700 20 reqs_thread_entry: start 2023-09-22T13:38:25.349 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.347+0000 7fc13fe4e500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:25.349 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.347+0000 7fc13fe4e500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:25.349 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.347+0000 7fc0577fe700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:55 2023 2023-09-22T13:38:25.349 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:25.349 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.347+0000 7fc056ffd700 20 reqs_thread_entry: start 2023-09-22T13:38:25.349 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.347+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb350e0 obj=default.rgw.meta:root:check-bucket state=0x55f7da6d40c0 s->prefetch_data=0 2023-09-22T13:38:25.350 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.347+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.350 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.347+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.350 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.347+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:25.350 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.347+0000 7fc12b330700 0 -- 172.21.15.69:0/1048145908 >> [v2:172.21.15.69:6808/145166,v1:172.21.15.69:6809/145166] conn(0x55f7da7c5fb0 msgr2=0x55f7da7c8400 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:25.350 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.347+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb350e0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.350 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.348+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.350 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.348+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.350 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.348+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.350 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.348+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.351 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.349+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.349+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.349+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb34eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.349+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.349+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.349+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.349+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.349+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.349+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.350+0000 7fc13fe4e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:25.352 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.350+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb34eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.353 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.350+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.353 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.350+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.353 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.350+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.353 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.350+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.353 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.351+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.353 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.351+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.351+0000 7fc13fe4e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:25.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.351+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb34eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.352+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.352+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.352+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.352+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.352+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.352+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.352+0000 7fc13fe4e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:25.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.353+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb34eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.356 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.353+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.356 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.353+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.356 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.353+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.356 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.353+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.356 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.353+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.356 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.353+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.356 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.353+0000 7fc13fe4e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:25.356 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.354+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb34eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.355+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.355+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.355+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.355+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.355+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.355+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.355+0000 7fc13fe4e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:25.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.356+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb34eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.356+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.356+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.356+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.356+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.357+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.357+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.357+0000 7fc13fe4e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:25.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.357+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb34eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.358+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.358+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.358+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.358+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.358+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.358+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.358+0000 7fc13fe4e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:25.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.359+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb34eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.359+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.359+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.359+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.359+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.359+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.359+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.359+0000 7fc13fe4e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:25.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.360+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb34eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.360+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.360+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.360+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.360+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.363 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.360+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.363 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.360+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.363 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.360+0000 7fc13fe4e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:25.363 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.361+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb34eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.361+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.361+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.361+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.361+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.362+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.362+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.362+0000 7fc13fe4e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:25.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.362+0000 7fc13fe4e500 20 get_system_obj_state: rctx=0x7ffe0eb34eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f7da656810 s->prefetch_data=0 2023-09-22T13:38:25.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.363+0000 7fc13fe4e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:25.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.363+0000 7fc13fe4e500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:25.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.363+0000 7fc13fe4e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:25.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.363+0000 7fc13fe4e500 20 rados->read ofs=0 len=0 2023-09-22T13:38:25.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.363+0000 7fc13fe4e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:25.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.363+0000 7fc13fe4e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:25.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.363+0000 7fc13fe4e500 20 bucket index oid: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:25.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.365+0000 7fc13fe4e500 20 remove_watcher() i=0 2023-09-22T13:38:25.366 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.365+0000 7fc13fe4e500 2 removed watcher, disabling cache 2023-09-22T13:38:25.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.366+0000 7fc13fe4e500 20 remove_watcher() i=1 2023-09-22T13:38:25.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.367+0000 7fc13fe4e500 20 remove_watcher() i=2 2023-09-22T13:38:25.372 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.371+0000 7fc13fe4e500 20 remove_watcher() i=3 2023-09-22T13:38:25.374 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.372+0000 7fc13fe4e500 20 remove_watcher() i=4 2023-09-22T13:38:25.375 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.373+0000 7fc13fe4e500 20 remove_watcher() i=5 2023-09-22T13:38:25.376 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.375+0000 7fc13fe4e500 20 remove_watcher() i=6 2023-09-22T13:38:25.377 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.376+0000 7fc13fe4e500 20 remove_watcher() i=7 2023-09-22T13:38:25.378 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:25.376+0000 7fc0fcfe1700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:25.389 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:25.389 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:25.389 INFO:tasks.workunit.client.0.smithi069.stderr:ceph config rm client rgw_debug_inject_set_olh_err 2023-09-22T13:38:25.777 INFO:tasks.workunit.client.0.smithi069.stderr:ceph config rm client rgw_debug_inject_olh_cancel_modification_err 2023-09-22T13:38:26.166 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bucket check --fix --bucket check-bucket 2023-09-22T13:38:26.190 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:26.190 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:26.197 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.196+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.199 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.198+0000 7f5aca190500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.199 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.198+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.200 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.198+0000 7f5aca190500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.200 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.198+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.200 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.198+0000 7f5aca190500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.200 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.198+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.201 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.199+0000 7f5aca190500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.213 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.211+0000 7f5aca190500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:26.213 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.211+0000 7f5aca190500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:26.213 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.211+0000 7f5aca190500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:26.213 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.211+0000 7f5aca190500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:26.213 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.211+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.213 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.212+0000 7f5aca190500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.214 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.212+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.215 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.212+0000 7f5aca190500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:26.216 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.212+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.216 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.212+0000 7f5aca190500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.216 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.212+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.216 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.213+0000 7f5aca190500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:26.216 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.213+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.216 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.213+0000 7f5aca190500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.216 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.213+0000 7f5aca190500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:26.216 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.213+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.216 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.213+0000 7f5aca190500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.216 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.213+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.217 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.213+0000 7f5aca190500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.217 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.213+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.217 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.214+0000 7f5aca190500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:26.217 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.214+0000 7f5aca190500 20 zonegroup default 2023-09-22T13:38:26.217 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.214+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.217 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.214+0000 7f5aca190500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.217 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.214+0000 7f5aca190500 10 Cannot find current period zone using local zone 2023-09-22T13:38:26.217 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.214+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.217 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.214+0000 7f5aca190500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.217 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.214+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.218 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.214+0000 7f5aca190500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.219 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.214+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.219 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.215+0000 7f5aca190500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:26.219 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.215+0000 7f5aca190500 20 zone default found 2023-09-22T13:38:26.219 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.215+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.219 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.215+0000 7f5aca190500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.219 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.215+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.219 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.215+0000 7f5aca190500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.219 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.215+0000 7f5aca190500 20 started sync module instance, tier type = 2023-09-22T13:38:26.219 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.215+0000 7f5aca190500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:26.235 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.233+0000 7f5aca190500 20 add_watcher() i=0 2023-09-22T13:38:26.235 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.233+0000 7f5aca190500 20 add_watcher() i=1 2023-09-22T13:38:26.235 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.233+0000 7f5aca190500 20 add_watcher() i=2 2023-09-22T13:38:26.235 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.233+0000 7f5aca190500 20 add_watcher() i=3 2023-09-22T13:38:26.235 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.233+0000 7f5aca190500 20 add_watcher() i=4 2023-09-22T13:38:26.236 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.233+0000 7f5aca190500 20 add_watcher() i=5 2023-09-22T13:38:26.236 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.233+0000 7f5aca190500 20 add_watcher() i=6 2023-09-22T13:38:26.238 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.236+0000 7f5aca190500 20 add_watcher() i=7 2023-09-22T13:38:26.238 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.236+0000 7f5aca190500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:26.238 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.236+0000 7f5ab5e73700 0 -- 172.21.15.69:0/2256876441 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x563b31c50dc0 msgr2=0x563b31c531b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:38:26.240 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.238+0000 7f5a867e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:26.240 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.238+0000 7f5aca190500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:26.240 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.238+0000 7f5aca190500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:26.244 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.242+0000 7f5ab4e71700 0 -- 172.21.15.69:0/1560099695 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x563b31c97bb0 msgr2=0x563b31c9a5a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:26.246 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.244+0000 7f5aca190500 5 note: GC not initialized 2023-09-22T13:38:26.248 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.245+0000 7f59f57fa700 20 reqs_thread_entry: start 2023-09-22T13:38:26.248 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.245+0000 7f59e6ffd700 20 reqs_thread_entry: start 2023-09-22T13:38:26.248 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.245+0000 7f5aca190500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:26.248 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:26.248 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f59e57fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:56 2023 2023-09-22T13:38:26.249 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:26.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f59e4ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:26.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 20 get_system_obj_state: rctx=0x7fff00c494a8 obj=default.rgw.meta:root:check-bucket state=0x563b31c36720 s->prefetch_data=0 2023-09-22T13:38:26.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:38:26.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:38:26.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:26.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:26.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:26.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:38:26.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.249 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:26.250 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:38:26.250 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:26.250 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 20 get_system_obj_state: rctx=0x7fff00c49578 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x563b31c36720 s->prefetch_data=0 2023-09-22T13:38:26.250 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.246+0000 7f5aca190500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : miss 2023-09-22T13:38:26.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x16 2023-09-22T13:38:26.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:26.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:26.251 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x13 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 2023-09-22T13:38:26.252 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 20 RGWRados::Bucket::List::list_objects_ordered starting attempt 1 2023-09-22T13:38:26.253 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.247+0000 7f5aca190500 10 RGWRados::cls_bucket_list_ordered: :check-bucket[f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1]) start_after="_multipart_[]", prefix="_multipart_" num_entries=1001, list_versions=1, expansion_factor=1 2023-09-22T13:38:26.253 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.248+0000 7f5aca190500 10 RGWRados::cls_bucket_list_ordered request from each of 11 shard(s) for 112 entries to get 1001 total entries 2023-09-22T13:38:26.253 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.248+0000 7f5aca190500 20 RGWRados::cls_bucket_list_ordered: returning, count=0, is_truncated=0 2023-09-22T13:38:26.253 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.248+0000 7f5aca190500 20 RGWRados::cls_bucket_list_ordered: returning, last_entry NOT SET 2023-09-22T13:38:26.253 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.248+0000 7f5aca190500 20 RGWRados::Bucket::List::list_objects_ordered INFO end of outer loop, truncated=0, count=0, attempt=1 2023-09-22T13:38:26.260 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.259+0000 7f5aca190500 20 remove_watcher() i=0 2023-09-22T13:38:26.260 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.259+0000 7f5aca190500 2 removed watcher, disabling cache 2023-09-22T13:38:26.261 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.260+0000 7f5aca190500 20 remove_watcher() i=1 2023-09-22T13:38:26.262 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.261+0000 7f5aca190500 20 remove_watcher() i=2 2023-09-22T13:38:26.263 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.262+0000 7f5aca190500 20 remove_watcher() i=3 2023-09-22T13:38:26.265 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.263+0000 7f5aca190500 20 remove_watcher() i=4 2023-09-22T13:38:26.266 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.264+0000 7f5aca190500 20 remove_watcher() i=5 2023-09-22T13:38:26.267 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.265+0000 7f5aca190500 20 remove_watcher() i=6 2023-09-22T13:38:26.268 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.266+0000 7f5aca190500 20 remove_watcher() i=7 2023-09-22T13:38:26.269 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.267+0000 7f5a867e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:26.275 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys 2023-09-22T13:38:26.303 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:26.303 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:26.311 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.309+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.311+0000 7f619aa23500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.311+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.313 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.311+0000 7f619aa23500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.313 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.311+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.313 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.312+0000 7f619aa23500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.313 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.312+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.314 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.313+0000 7f619aa23500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.323+0000 7f619aa23500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:26.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.323+0000 7f619aa23500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:26.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.323+0000 7f619aa23500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:26.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.323+0000 7f619aa23500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:26.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.323+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.323+0000 7f619aa23500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.323+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.324+0000 7f619aa23500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:26.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.324+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.324+0000 7f619aa23500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.324+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.324+0000 7f619aa23500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:26.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.324+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.324+0000 7f619aa23500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.324+0000 7f619aa23500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.324+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.325+0000 7f619aa23500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.325+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.325+0000 7f619aa23500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.325+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.325+0000 7f619aa23500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.325+0000 7f619aa23500 20 zonegroup default 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.325+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.325+0000 7f619aa23500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.325+0000 7f619aa23500 10 Cannot find current period zone using local zone 2023-09-22T13:38:26.328 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.325+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.326+0000 7f619aa23500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.326+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.326+0000 7f619aa23500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.326+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.327+0000 7f619aa23500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:26.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.327+0000 7f619aa23500 20 zone default found 2023-09-22T13:38:26.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.327+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.327+0000 7f619aa23500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.327+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.330 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.327+0000 7f619aa23500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.327+0000 7f619aa23500 20 started sync module instance, tier type = 2023-09-22T13:38:26.331 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.327+0000 7f619aa23500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:26.341 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.339+0000 7f619aa23500 20 add_watcher() i=0 2023-09-22T13:38:26.341 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.339+0000 7f619aa23500 20 add_watcher() i=1 2023-09-22T13:38:26.341 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.339+0000 7f619aa23500 20 add_watcher() i=2 2023-09-22T13:38:26.341 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.339+0000 7f619aa23500 20 add_watcher() i=3 2023-09-22T13:38:26.341 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.339+0000 7f619aa23500 20 add_watcher() i=4 2023-09-22T13:38:26.341 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.339+0000 7f619aa23500 20 add_watcher() i=5 2023-09-22T13:38:26.344 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.342+0000 7f6185f05700 0 -- 172.21.15.69:0/824746852 >> [v2:172.21.15.69:6808/145166,v1:172.21.15.69:6809/145166] conn(0x55f852370370 msgr2=0x55f8523727f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:26.344 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.342+0000 7f619aa23500 20 add_watcher() i=6 2023-09-22T13:38:26.344 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.342+0000 7f619aa23500 20 add_watcher() i=7 2023-09-22T13:38:26.344 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.342+0000 7f619aa23500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:26.345 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.344+0000 7f61577de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:26.346 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.344+0000 7f619aa23500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:26.346 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.344+0000 7f619aa23500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:26.349 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.348+0000 7f6186706700 0 -- 172.21.15.69:0/1172049385 >> [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] conn(0x55f8523aa060 msgr2=0x55f85227bd20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:26.351 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.350+0000 7f619aa23500 5 note: GC not initialized 2023-09-22T13:38:26.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.351+0000 7f60c67fc700 20 reqs_thread_entry: start 2023-09-22T13:38:26.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.351+0000 7f60b7fff700 20 reqs_thread_entry: start 2023-09-22T13:38:26.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.351+0000 7f619aa23500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:26.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.351+0000 7f619aa23500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:26.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.351+0000 7f60b67fc700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:56 2023 2023-09-22T13:38:26.354 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:26.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.351+0000 7f60b5ffb700 20 reqs_thread_entry: start 2023-09-22T13:38:26.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 20 get_system_obj_state: rctx=0x7ffd06db2018 obj=default.rgw.meta:root:check-bucket state=0x55f8523882e0 s->prefetch_data=0 2023-09-22T13:38:26.354 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:38:26.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:38:26.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:26.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:26.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:26.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:38:26.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:26.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:38:26.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:38:26.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 20 get_system_obj_state: rctx=0x7ffd06db20e8 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x55f8523882e0 s->prefetch_data=0 2023-09-22T13:38:26.355 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.352+0000 7f619aa23500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : miss 2023-09-22T13:38:26.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x16 2023-09-22T13:38:26.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:26.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:26.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:26.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:26.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:26.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:26.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:38:26.357 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:26.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 info.flags=0x13 2023-09-22T13:38:26.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 to cache LRU end 2023-09-22T13:38:26.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:38:26.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:38:26.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 2023-09-22T13:38:26.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.353+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.0 2023-09-22T13:38:26.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.354+0000 7f619aa23500 1 NOTICE: finished shard 0 (0 entries removed) 2023-09-22T13:38:26.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.354+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:26.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.354+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.2 2023-09-22T13:38:26.358 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.355+0000 7f619aa23500 1 NOTICE: finished shard 2 (0 entries removed) 2023-09-22T13:38:26.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.355+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.3 2023-09-22T13:38:26.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.355+0000 7f619aa23500 1 NOTICE: finished shard 3 (0 entries removed) 2023-09-22T13:38:26.359 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.355+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.4 2023-09-22T13:38:26.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.356+0000 7f619aa23500 1 NOTICE: finished shard 4 (0 entries removed) 2023-09-22T13:38:26.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.356+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:26.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.356+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.6 2023-09-22T13:38:26.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.357+0000 7f619aa23500 1 NOTICE: finished shard 6 (0 entries removed) 2023-09-22T13:38:26.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.357+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.7 2023-09-22T13:38:26.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.357+0000 7f619aa23500 1 NOTICE: finished shard 7 (0 entries removed) 2023-09-22T13:38:26.360 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.357+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:26.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.357+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.9 2023-09-22T13:38:26.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.358+0000 7f619aa23500 1 NOTICE: finished shard 9 (0 entries removed) 2023-09-22T13:38:26.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.358+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:26.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.358+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:26.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.359+0000 7f619aa23500 20 get_obj_state: rctx=0x55f852454670 obj=check-bucket:e state=0x55f852479df8 s->prefetch_data=0 2023-09-22T13:38:26.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.359+0000 7f619aa23500 20 get_obj_state: setting s->obj_tag to riim9c35dmb1mojo1hqfhk9ja31j4q3y 2023-09-22T13:38:26.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.359+0000 7f619aa23500 20 get_obj_state_impl: setting s->olh_tag to cvelxjfuwpwr2z8vtp07vytwq3qyqpm3 2023-09-22T13:38:26.361 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.359+0000 7f619aa23500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=cvelxjfuwpwr2z8vtp07vytwq3qyqpm3 2023-09-22T13:38:26.362 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.360+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:26.363 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.362+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:26.363 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.362+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:26.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.362+0000 7f619aa23500 20 olh_log_entry: epoch=2 op=2 key=e[] 2023-09-22T13:38:26.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.362+0000 7f619aa23500 20 olh_log_entry: epoch=2 op=3 key=e[AJo34k315jYv2qDHhEQRRScYG3QTgS1] 2023-09-22T13:38:26.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.362+0000 7f619aa23500 20 get_obj_state: rctx=0x55f852454670 obj=check-bucket:_:AJo34k315jYv2qDHhEQRRScYG3QTgS1_e state=0x55f8524c21f8 s->prefetch_data=0 2023-09-22T13:38:26.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.362+0000 7f619aa23500 10 manifest: total_size = 9 2023-09-22T13:38:26.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.362+0000 7f619aa23500 20 get_obj_state: setting s->obj_tag to f54d190c-30d0-4b87-88c4-5bcc7de96de4.4553.629518562006048660 2023-09-22T13:38:26.364 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.362+0000 7f619aa23500 20 get_obj_state: rctx=0x55f852454670 obj=check-bucket:_:AJo34k315jYv2qDHhEQRRScYG3QTgS1_e state=0x55f8524c21f8 s->prefetch_data=0 2023-09-22T13:38:26.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.362+0000 7f619aa23500 20 prepare_atomic_modification: state is not atomic. state=0x55f8524c21f8 2023-09-22T13:38:26.365 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.362+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:26.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.365+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : miss 2023-09-22T13:38:26.367 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.365+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.366+0000 7f619aa23500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.366+0000 7f619aa23500 10 cache put: name=default.rgw.log++bucket.sync-source-hints.check-bucket info.flags=0x0 2023-09-22T13:38:26.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.366+0000 7f619aa23500 10 adding default.rgw.log++bucket.sync-source-hints.check-bucket to cache LRU end 2023-09-22T13:38:26.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.366+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : miss 2023-09-22T13:38:26.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.366+0000 7f619aa23500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.366+0000 7f619aa23500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.366+0000 7f619aa23500 10 cache put: name=default.rgw.log++bucket.sync-target-hints.check-bucket info.flags=0x0 2023-09-22T13:38:26.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.366+0000 7f619aa23500 10 adding default.rgw.log++bucket.sync-target-hints.check-bucket to cache LRU end 2023-09-22T13:38:26.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.366+0000 7f619aa23500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:26.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.366+0000 7f619aa23500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:26.368 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.366+0000 7f619aa23500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:26.372 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.371+0000 7f619aa23500 20 get_obj_state: rctx=0x55f852454670 obj=check-bucket:e state=0x55f852479df8 s->prefetch_data=0 2023-09-22T13:38:26.372 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.371+0000 7f619aa23500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=cvelxjfuwpwr2z8vtp07vytwq3qyqpm3 2023-09-22T13:38:26.379 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.377+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:26.380 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.378+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.1 2023-09-22T13:38:26.380 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.378+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.380 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.378+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.378+0000 7f619aa23500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:26.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.378+0000 7f619aa23500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:26.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.378+0000 7f619aa23500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:26.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.379+0000 7f619aa23500 1 NOTICE: finished shard 1 (1 entries removed) 2023-09-22T13:38:26.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.379+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:26.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.379+0000 7f619aa23500 20 get_obj_state: rctx=0x55f852478b70 obj=check-bucket:d state=0x55f852479df8 s->prefetch_data=0 2023-09-22T13:38:26.381 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.379+0000 7f619aa23500 20 get_obj_state: setting s->obj_tag to 7og6jayrgl9pa3zj0sbkaczgtjhhoces 2023-09-22T13:38:26.382 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.379+0000 7f619aa23500 20 get_obj_state_impl: setting s->olh_tag to 9swmaxhiqq1u6rb6yc6f3l2ygkkn61v3 2023-09-22T13:38:26.382 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.379+0000 7f619aa23500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=9swmaxhiqq1u6rb6yc6f3l2ygkkn61v3 2023-09-22T13:38:26.382 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.381+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:26.386 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.382+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:26.386 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.382+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:26.386 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.382+0000 7f619aa23500 20 olh_log_entry: epoch=3 op=3 key=d[J.Ds4KTNWmuEcnDvDdMuNL94p2kLBve] 2023-09-22T13:38:26.386 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.382+0000 7f619aa23500 20 get_obj_state: rctx=0x55f852478b70 obj=check-bucket:_:J.Ds4KTNWmuEcnDvDdMuNL94p2kLBve_d state=0x55f8524c21f8 s->prefetch_data=0 2023-09-22T13:38:26.387 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.383+0000 7f619aa23500 10 manifest: total_size = 11 2023-09-22T13:38:26.387 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.383+0000 7f619aa23500 20 get_obj_state: setting s->obj_tag to f54d190c-30d0-4b87-88c4-5bcc7de96de4.4553.846404625087403209 2023-09-22T13:38:26.387 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.383+0000 7f619aa23500 20 get_obj_state: rctx=0x55f852478b70 obj=check-bucket:_:J.Ds4KTNWmuEcnDvDdMuNL94p2kLBve_d state=0x55f8524c21f8 s->prefetch_data=0 2023-09-22T13:38:26.387 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.383+0000 7f619aa23500 20 prepare_atomic_modification: state is not atomic. state=0x55f8524c21f8 2023-09-22T13:38:26.387 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.383+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:26.387 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.385+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.387 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.385+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.388 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.385+0000 7f619aa23500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:26.388 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.385+0000 7f619aa23500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:26.388 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.385+0000 7f619aa23500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:26.388 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.387+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:26.389 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.387+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:26.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.389+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.5 2023-09-22T13:38:26.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.389+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.389+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.389+0000 7f619aa23500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:26.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.389+0000 7f619aa23500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:26.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.389+0000 7f619aa23500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:26.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.389+0000 7f619aa23500 1 NOTICE: finished shard 5 (1 entries removed) 2023-09-22T13:38:26.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.389+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:26.391 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.389+0000 7f619aa23500 20 get_obj_state: rctx=0x55f852499e70 obj=check-bucket:c state=0x55f8524c21f8 s->prefetch_data=0 2023-09-22T13:38:26.392 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.390+0000 7f619aa23500 20 get_obj_state: setting s->obj_tag to k8exnn0n50z900zh3gpba2vguuni2vv3 2023-09-22T13:38:26.392 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.390+0000 7f619aa23500 20 get_obj_state_impl: setting s->olh_tag to p8q1z307e1zm9cdyg6bf2k016jkuepea 2023-09-22T13:38:26.392 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.390+0000 7f619aa23500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=p8q1z307e1zm9cdyg6bf2k016jkuepea 2023-09-22T13:38:26.393 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.391+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:26.396 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.395+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:26.396 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.395+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:26.397 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.395+0000 7f619aa23500 20 olh_log_entry: epoch=3 op=3 key=c[MkV938byBZzO4QxHj8ocfy49u9FzneT] 2023-09-22T13:38:26.397 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.395+0000 7f619aa23500 20 get_obj_state: rctx=0x55f852499e70 obj=check-bucket:_:MkV938byBZzO4QxHj8ocfy49u9FzneT_c state=0x55f852479d18 s->prefetch_data=0 2023-09-22T13:38:26.397 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.395+0000 7f619aa23500 10 manifest: total_size = 9 2023-09-22T13:38:26.397 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.395+0000 7f619aa23500 20 get_obj_state: setting s->obj_tag to f54d190c-30d0-4b87-88c4-5bcc7de96de4.4553.7029211546435408514 2023-09-22T13:38:26.397 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.395+0000 7f619aa23500 20 get_obj_state: rctx=0x55f852499e70 obj=check-bucket:_:MkV938byBZzO4QxHj8ocfy49u9FzneT_c state=0x55f852479d18 s->prefetch_data=0 2023-09-22T13:38:26.397 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.395+0000 7f619aa23500 20 prepare_atomic_modification: state is not atomic. state=0x55f852479d18 2023-09-22T13:38:26.397 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.395+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:26.401 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.399+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.401 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.399+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.401 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.399+0000 7f619aa23500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:26.401 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.399+0000 7f619aa23500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:26.401 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.399+0000 7f619aa23500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:26.405 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.403+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:26.405 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.403+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:26.411 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.409+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.8 2023-09-22T13:38:26.411 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.409+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.411 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.409+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.411 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.409+0000 7f619aa23500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:26.411 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.409+0000 7f619aa23500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:26.411 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.409+0000 7f619aa23500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:26.411 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.409+0000 7f619aa23500 1 NOTICE: finished shard 8 (1 entries removed) 2023-09-22T13:38:26.411 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.409+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:26.411 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.409+0000 7f619aa23500 20 get_obj_state: rctx=0x55f8524b9e70 obj=check-bucket:f state=0x55f8524c21f8 s->prefetch_data=0 2023-09-22T13:38:26.412 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.410+0000 7f619aa23500 20 get_obj_state: setting s->obj_tag to 5oi1jox2fkgtu0rk0kdsd60joyqnpbh7 2023-09-22T13:38:26.412 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.410+0000 7f619aa23500 20 get_obj_state_impl: setting s->olh_tag to rmrs5agi0s1wo2u3u6iypgttlwe9w572 2023-09-22T13:38:26.412 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.410+0000 7f619aa23500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=rmrs5agi0s1wo2u3u6iypgttlwe9w572 2023-09-22T13:38:26.413 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.411+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:26.414 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.413+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:26.414 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.413+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:26.414 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.413+0000 7f619aa23500 20 olh_log_entry: epoch=2 op=2 key=f[] 2023-09-22T13:38:26.415 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.413+0000 7f619aa23500 20 olh_log_entry: epoch=2 op=3 key=f[vdlazyakSx9XxbhMK.6nVoe2YFv5Ixb] 2023-09-22T13:38:26.415 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.413+0000 7f619aa23500 20 get_obj_state: rctx=0x55f8524b9e70 obj=check-bucket:_:vdlazyakSx9XxbhMK.6nVoe2YFv5Ixb_f state=0x55f852436148 s->prefetch_data=0 2023-09-22T13:38:26.415 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.413+0000 7f619aa23500 10 manifest: total_size = 22 2023-09-22T13:38:26.415 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.413+0000 7f619aa23500 20 get_obj_state: setting s->obj_tag to f54d190c-30d0-4b87-88c4-5bcc7de96de4.4553.7843226506675958380 2023-09-22T13:38:26.415 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.413+0000 7f619aa23500 20 get_obj_state: rctx=0x55f8524b9e70 obj=check-bucket:_:vdlazyakSx9XxbhMK.6nVoe2YFv5Ixb_f state=0x55f852436148 s->prefetch_data=0 2023-09-22T13:38:26.416 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.413+0000 7f619aa23500 20 prepare_atomic_modification: state is not atomic. state=0x55f852436148 2023-09-22T13:38:26.416 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.413+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:26.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.416+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.416+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.417 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.416+0000 7f619aa23500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:26.418 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.416+0000 7f619aa23500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:26.418 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.416+0000 7f619aa23500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:26.419 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.417+0000 7f619aa23500 20 get_obj_state: rctx=0x55f8524b9e70 obj=check-bucket:f state=0x55f8524c21f8 s->prefetch_data=0 2023-09-22T13:38:26.419 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.417+0000 7f619aa23500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=rmrs5agi0s1wo2u3u6iypgttlwe9w572 2023-09-22T13:38:26.426 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.424+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:26.427 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.426+0000 7f619aa23500 20 bucket index object: default.rgw.buckets.index:.dir.f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1.10 2023-09-22T13:38:26.428 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.426+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.428 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.426+0000 7f619aa23500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:38:26.428 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.426+0000 7f619aa23500 10 chain_cache_entry: cache_locator= 2023-09-22T13:38:26.428 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.426+0000 7f619aa23500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:38:26.428 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.426+0000 7f619aa23500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:38:26.428 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.426+0000 7f619aa23500 1 NOTICE: finished shard 10 (1 entries removed) 2023-09-22T13:38:26.428 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.426+0000 7f619aa23500 1 NOTICE: finished all shards (4 entries removed) 2023-09-22T13:38:26.432 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.431+0000 7f619aa23500 20 remove_watcher() i=0 2023-09-22T13:38:26.432 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.431+0000 7f619aa23500 2 removed watcher, disabling cache 2023-09-22T13:38:26.433 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.432+0000 7f619aa23500 20 remove_watcher() i=1 2023-09-22T13:38:26.434 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.433+0000 7f619aa23500 20 remove_watcher() i=2 2023-09-22T13:38:26.435 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.434+0000 7f619aa23500 20 remove_watcher() i=3 2023-09-22T13:38:26.436 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.435+0000 7f619aa23500 20 remove_watcher() i=4 2023-09-22T13:38:26.437 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.436+0000 7f619aa23500 20 remove_watcher() i=5 2023-09-22T13:38:26.438 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.437+0000 7f619aa23500 20 remove_watcher() i=6 2023-09-22T13:38:26.439 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.438+0000 7f619aa23500 20 remove_watcher() i=7 2023-09-22T13:38:26.440 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.438+0000 7f61577de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:26.680 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:26.680 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:26.680 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:26.680 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:26.680 INFO:tasks.workunit.client.0.smithi069.stderr:radosgw-admin bucket stats --bucket check-bucket 2023-09-22T13:38:26.705 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:38:26.705 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:38:26.717 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.715+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.719 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.717+0000 7f39a1038500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.719 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.717+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.720 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.718+0000 7f39a1038500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.720 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.718+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.720 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.718+0000 7f39a1038500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.720 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.718+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.720 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.719+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.730 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.729+0000 7f39a1038500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:38:26.731 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.729+0000 7f39a1038500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:38:26.731 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.729+0000 7f39a1038500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:38:26.731 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.729+0000 7f39a1038500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:38:26.731 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.729+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.732 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.729+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.732 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.729+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.732 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.730+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:26.732 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.730+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.732 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.730+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.732 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.730+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.733 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.730+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:26.733 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.730+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.733 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.730+0000 7f39a1038500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.733 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.730+0000 7f39a1038500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:38:26.733 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.730+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.733 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.731+0000 7f39a1038500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.733 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.731+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.734 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.731+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.734 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.731+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.735 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.731+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:38:26.735 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.731+0000 7f39a1038500 20 zonegroup default 2023-09-22T13:38:26.735 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.731+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.735 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.731+0000 7f39a1038500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.735 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.731+0000 7f39a1038500 10 Cannot find current period zone using local zone 2023-09-22T13:38:26.735 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.731+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.735 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.732+0000 7f39a1038500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.735 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.732+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.735 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.732+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:38:26.735 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.732+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.736 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.732+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:38:26.736 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.732+0000 7f39a1038500 20 zone default found 2023-09-22T13:38:26.736 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.732+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.736 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.732+0000 7f39a1038500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.736 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.733+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.736 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.733+0000 7f39a1038500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:38:26.736 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.733+0000 7f39a1038500 20 started sync module instance, tier type = 2023-09-22T13:38:26.736 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.733+0000 7f39a1038500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:38:26.755 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.754+0000 7f39a1038500 20 add_watcher() i=0 2023-09-22T13:38:26.755 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.754+0000 7f39a1038500 20 add_watcher() i=1 2023-09-22T13:38:26.756 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.754+0000 7f39a1038500 20 add_watcher() i=2 2023-09-22T13:38:26.756 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.754+0000 7f39a1038500 20 add_watcher() i=3 2023-09-22T13:38:26.756 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.754+0000 7f39a1038500 20 add_watcher() i=4 2023-09-22T13:38:26.756 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.754+0000 7f39a1038500 20 add_watcher() i=5 2023-09-22T13:38:26.756 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.754+0000 7f39a1038500 20 add_watcher() i=6 2023-09-22T13:38:26.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.757+0000 7f39a1038500 20 add_watcher() i=7 2023-09-22T13:38:26.759 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.757+0000 7f39a1038500 2 all 8 watchers are set, enabling cache 2023-09-22T13:38:26.764 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.762+0000 7f398cd1b700 0 -- 172.21.15.69:0/61378988 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x5649eba05970 msgr2=0x5649eba07d60 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:26.764 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.763+0000 7f395dfe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:26.765 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.763+0000 7f39a1038500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:38:26.765 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.763+0000 7f39a1038500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:38:26.768 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.767+0000 7f398c51a700 0 -- 172.21.15.69:0/439574900 >> [v2:172.21.15.69:3301/0,v1:172.21.15.69:6790/0] conn(0x5649eba447e0 msgr2=0x5649eba3bc40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:38:26.771 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.770+0000 7f39a1038500 5 note: GC not initialized 2023-09-22T13:38:26.772 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.771+0000 7f38d4ff9700 20 reqs_thread_entry: start 2023-09-22T13:38:26.773 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.771+0000 7f38c27fc700 20 reqs_thread_entry: start 2023-09-22T13:38:26.773 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.771+0000 7f39a1038500 20 init_complete bucket index max shards: 11 2023-09-22T13:38:26.773 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.771+0000 7f39a1038500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:38:26.773 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.771+0000 7f38c0ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:38:57 2023 2023-09-22T13:38:26.773 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:38:26.773 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.771+0000 7f39a1038500 20 get_system_obj_state: rctx=0x7fff56b26c48 obj=default.rgw.meta:root:check-bucket state=0x5649eb891810 s->prefetch_data=0 2023-09-22T13:38:26.773 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.771+0000 7f38abfff700 20 reqs_thread_entry: start 2023-09-22T13:38:26.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.771+0000 7f39a1038500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:26.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.771+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.772+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:26.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.772+0000 7f39a1038500 20 get_system_obj_state: rctx=0x7fff56b26d18 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x5649eb891810 s->prefetch_data=0 2023-09-22T13:38:26.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.772+0000 7f39a1038500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:26.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.772+0000 7f39a1038500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:26.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.772+0000 7f39a1038500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:26.774 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.772+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.773+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:26.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.773+0000 7f39a1038500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:26.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.773+0000 7f39a1038500 20 get_system_obj_state: rctx=0x7fff56b27010 obj=default.rgw.meta:root:check-bucket state=0x5649eb891810 s->prefetch_data=0 2023-09-22T13:38:26.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.773+0000 7f39a1038500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:26.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.773+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.775 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.774+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:38:26.776 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.774+0000 7f39a1038500 20 get_system_obj_state: rctx=0x7fff56b27010 obj=default.rgw.meta:root:.bucket.meta.check-bucket:f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.1 state=0x5649eb84b910 s->prefetch_data=0 2023-09-22T13:38:26.776 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.774+0000 7f39a1038500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:38:26.776 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.774+0000 7f39a1038500 20 Read xattr: user.rgw.acl 2023-09-22T13:38:26.776 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.774+0000 7f39a1038500 20 Read xattr: user.rgw.idtag 2023-09-22T13:38:26.776 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.774+0000 7f39a1038500 20 rados->read ofs=0 len=0 2023-09-22T13:38:26.776 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.774+0000 7f39a1038500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:38:26.777 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.774+0000 7f39a1038500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:38:26.778 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.777+0000 7f39a1038500 20 remove_watcher() i=0 2023-09-22T13:38:26.778 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.777+0000 7f39a1038500 2 removed watcher, disabling cache 2023-09-22T13:38:26.779 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.778+0000 7f39a1038500 20 remove_watcher() i=1 2023-09-22T13:38:26.781 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.779+0000 7f39a1038500 20 remove_watcher() i=2 2023-09-22T13:38:26.782 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.780+0000 7f39a1038500 20 remove_watcher() i=3 2023-09-22T13:38:26.783 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.781+0000 7f39a1038500 20 remove_watcher() i=4 2023-09-22T13:38:26.784 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.782+0000 7f39a1038500 20 remove_watcher() i=5 2023-09-22T13:38:26.786 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.785+0000 7f39a1038500 20 remove_watcher() i=6 2023-09-22T13:38:26.790 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.788+0000 7f39a1038500 20 remove_watcher() i=7 2023-09-22T13:38:26.791 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:38:26.789+0000 7f395dfe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:38:26.798 INFO:tasks.workunit.client.0.smithi069.stderr:{'rgw.main': {'size': 0, 'size_actual': 0, 'size_utilized': 0, 'size_kb': 0, 'size_kb_actual': 0, 'size_kb_utilized': 0, 'num_objects': 0}} 2023-09-22T13:38:26.798 INFO:tasks.workunit.client.0.smithi069.stderr:Deleting bucket check-bucket 2023-09-22T13:38:26.847 INFO:tasks.workunit.client.0.smithi069.stdout:Could not connect to the endpoint URL: "http://localhost:80/" 2023-09-22T13:38:26.847 INFO:tasks.workunit.client.0.smithi069.stdout:Could not connect to the endpoint URL: "http://localhost:8000/" 2023-09-22T13:38:26.847 INFO:tasks.workunit.client.0.smithi069.stdout:connected to https://localhost:443 2023-09-22T13:38:26.848 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:26.848 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:26.848 INFO:tasks.workunit.client.0.smithi069.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:38:26.848 INFO:tasks.workunit.client.0.smithi069.stderr: InsecureRequestWarning, 2023-09-22T13:38:26.848 INFO:tasks.workunit.client.0.smithi069.stderr:Completed bucket check tests 2023-09-22T13:38:26.888 INFO:tasks.workunit.client.0.smithi069.stderr:+ deactivate 2023-09-22T13:38:26.888 INFO:tasks.workunit.client.0.smithi069.stderr:+ '[' -n /home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin ']' 2023-09-22T13:38:26.889 INFO:tasks.workunit.client.0.smithi069.stderr:+ PATH=/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin 2023-09-22T13:38:26.889 INFO:tasks.workunit.client.0.smithi069.stderr:+ export PATH 2023-09-22T13:38:26.889 INFO:tasks.workunit.client.0.smithi069.stderr:+ unset _OLD_VIRTUAL_PATH 2023-09-22T13:38:26.889 INFO:tasks.workunit.client.0.smithi069.stderr:+ '[' -n '' ']' 2023-09-22T13:38:26.889 INFO:tasks.workunit.client.0.smithi069.stderr:+ '[' -n /usr/bin/bash -o -n '' ']' 2023-09-22T13:38:26.889 INFO:tasks.workunit.client.0.smithi069.stderr:+ hash -r 2023-09-22T13:38:26.889 INFO:tasks.workunit.client.0.smithi069.stderr:+ '[' -n '' ']' 2023-09-22T13:38:26.889 INFO:tasks.workunit.client.0.smithi069.stderr:+ unset VIRTUAL_ENV 2023-09-22T13:38:26.889 INFO:tasks.workunit.client.0.smithi069.stderr:+ '[' '!' '' = nondestructive ']' 2023-09-22T13:38:26.889 INFO:tasks.workunit.client.0.smithi069.stderr:+ unset -f deactivate 2023-09-22T13:38:26.889 INFO:tasks.workunit.client.0.smithi069.stderr:+ echo OK. 2023-09-22T13:38:26.890 INFO:tasks.workunit.client.0.smithi069.stdout:OK. 2023-09-22T13:38:26.890 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:38:26.890 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:38:26.958 INFO:tasks.workunit:Stopping ['rgw/run-bucket-check.sh'] on client.0... 2023-09-22T13:38:26.958 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-22T13:38:27.280 DEBUG:teuthology.parallel:result is None 2023-09-22T13:38:27.280 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:38:27.307 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:38:27.307 DEBUG:teuthology.orchestra.run.smithi069:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:38:27.503 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:38:27.503 INFO:teuthology.run_tasks:Running task workunit... 2023-09-22T13:38:27.512 INFO:tasks.workunit:Pulling workunits from ref 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:38:27.512 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-22T13:38:27.512 DEBUG:teuthology.orchestra.run.smithi069:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:38:27.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T13:38:27.532 INFO:teuthology.orchestra.run.smithi069.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-22T13:38:27.532 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:38:27.590 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-22T13:38:27.590 DEBUG:teuthology.orchestra.run.smithi069:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-22T13:38:27.649 INFO:tasks.workunit:timeout=3h 2023-09-22T13:38:27.649 INFO:tasks.workunit:cleanup=True 2023-09-22T13:38:27.649 DEBUG:teuthology.orchestra.run.smithi069:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:38:27.708 INFO:tasks.workunit.client.0.smithi069.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-22T13:39:26.684 INFO:tasks.workunit.client.0.smithi069.stderr:Note: switching to '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'. 2023-09-22T13:39:26.684 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:39:26.684 INFO:tasks.workunit.client.0.smithi069.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2023-09-22T13:39:26.684 INFO:tasks.workunit.client.0.smithi069.stderr:changes and commit them, and you can discard any commits you make in this 2023-09-22T13:39:26.684 INFO:tasks.workunit.client.0.smithi069.stderr:state without impacting any branches by switching back to a branch. 2023-09-22T13:39:26.685 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:39:26.685 INFO:tasks.workunit.client.0.smithi069.stderr:If you want to create a new branch to retain commits you create, you may 2023-09-22T13:39:26.685 INFO:tasks.workunit.client.0.smithi069.stderr:do so (now or later) by using -c with the switch command. Example: 2023-09-22T13:39:26.685 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:39:26.685 INFO:tasks.workunit.client.0.smithi069.stderr: git switch -c 2023-09-22T13:39:26.685 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:39:26.685 INFO:tasks.workunit.client.0.smithi069.stderr:Or undo this operation with: 2023-09-22T13:39:26.685 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:39:26.686 INFO:tasks.workunit.client.0.smithi069.stderr: git switch - 2023-09-22T13:39:26.686 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:39:26.686 INFO:tasks.workunit.client.0.smithi069.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2023-09-22T13:39:26.686 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:39:26.686 INFO:tasks.workunit.client.0.smithi069.stderr:HEAD is now at 31c126b7c1c rgw: radosgw-admin bucket check should only print index entries with --check-objects flag 2023-09-22T13:39:26.691 DEBUG:teuthology.orchestra.run.smithi069:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-22T13:39:26.749 INFO:tasks.workunit.client.0.smithi069.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-22T13:39:26.751 INFO:tasks.workunit.client.0.smithi069.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:39:26.751 INFO:tasks.workunit.client.0.smithi069.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-22T13:39:26.797 INFO:tasks.workunit.client.0.smithi069.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-22T13:39:26.844 INFO:tasks.workunit.client.0.smithi069.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-22T13:39:26.881 INFO:tasks.workunit.client.0.smithi069.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:39:26.882 INFO:tasks.workunit.client.0.smithi069.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:39:26.883 INFO:tasks.workunit.client.0.smithi069.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-22T13:39:26.919 INFO:tasks.workunit.client.0.smithi069.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:39:26.922 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:39:26.922 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-22T13:39:26.980 INFO:tasks.workunit:Running workunits matching cls/test_cls_lock.sh on client.0... 2023-09-22T13:39:26.981 INFO:tasks.workunit:Running workunit cls/test_cls_lock.sh... 2023-09-22T13:39:26.981 DEBUG:teuthology.orchestra.run.smithi069:workunit test cls/test_cls_lock.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_lock.sh 2023-09-22T13:39:27.051 INFO:tasks.workunit.client.0.smithi069.stdout:Running main() from gmock_main.cc 2023-09-22T13:39:27.051 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 11 tests from 1 test suite. 2023-09-22T13:39:27.051 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:39:27.051 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 11 tests from ClsLock 2023-09-22T13:39:27.052 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] ClsLock.TestMultiLocking 2023-09-22T13:39:28.606 INFO:tasks.workunit.client.0.smithi069.stdout:lock: mylock 2023-09-22T13:39:28.607 INFO:tasks.workunit.client.0.smithi069.stdout: lock_type: exclusive 2023-09-22T13:39:28.607 INFO:tasks.workunit.client.0.smithi069.stdout: tag: 2023-09-22T13:39:28.607 INFO:tasks.workunit.client.0.smithi069.stdout: lockers: 2023-09-22T13:39:28.607 INFO:tasks.workunit.client.0.smithi069.stdout: client.5115 expiration=2023-09-22T13:41:28.602604+0000 addr=v1:172.21.15.69:0/1068533783 cookie= 2023-09-22T13:39:28.610 INFO:tasks.workunit.client.0.smithi069.stdout:lock: mylock 2023-09-22T13:39:28.611 INFO:tasks.workunit.client.0.smithi069.stdout: lock_type: shared 2023-09-22T13:39:28.611 INFO:tasks.workunit.client.0.smithi069.stdout: tag: 2023-09-22T13:39:28.611 INFO:tasks.workunit.client.0.smithi069.stdout: lockers: 2023-09-22T13:39:28.611 INFO:tasks.workunit.client.0.smithi069.stdout: client.5115 expiration=2023-09-22T13:41:28.609364+0000 addr=v1:172.21.15.69:0/1068533783 cookie= 2023-09-22T13:39:28.611 INFO:tasks.workunit.client.0.smithi069.stdout: client.5121 expiration=0.000000 addr=v1:172.21.15.69:0/2688301401 cookie= 2023-09-22T13:39:28.612 INFO:tasks.workunit.client.0.smithi069.stdout:lock: mylock 2023-09-22T13:39:28.612 INFO:tasks.workunit.client.0.smithi069.stdout: lock_type: shared 2023-09-22T13:39:28.612 INFO:tasks.workunit.client.0.smithi069.stdout: tag: 2023-09-22T13:39:28.612 INFO:tasks.workunit.client.0.smithi069.stdout: lockers: 2023-09-22T13:39:28.612 INFO:tasks.workunit.client.0.smithi069.stdout: client.5121 expiration=0.000000 addr=v1:172.21.15.69:0/2688301401 cookie= 2023-09-22T13:39:29.584 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] ClsLock.TestMultiLocking (2533 ms) 2023-09-22T13:39:29.584 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] ClsLock.TestMeta 2023-09-22T13:39:31.623 INFO:tasks.workunit.client.0.smithi069.stdout:lock: mylock 2023-09-22T13:39:31.623 INFO:tasks.workunit.client.0.smithi069.stdout: lock_type: shared 2023-09-22T13:39:31.623 INFO:tasks.workunit.client.0.smithi069.stdout: tag: 2023-09-22T13:39:31.623 INFO:tasks.workunit.client.0.smithi069.stdout: lockers: 2023-09-22T13:39:31.623 INFO:tasks.workunit.client.0.smithi069.stdout: client.5133 expiration=0.000000 addr=v1:172.21.15.69:0/40886878 cookie= 2023-09-22T13:39:31.625 INFO:tasks.workunit.client.0.smithi069.stdout:lock: mylock 2023-09-22T13:39:31.625 INFO:tasks.workunit.client.0.smithi069.stdout: lock_type: exclusive 2023-09-22T13:39:31.625 INFO:tasks.workunit.client.0.smithi069.stdout: tag: new_tag 2023-09-22T13:39:31.625 INFO:tasks.workunit.client.0.smithi069.stdout: lockers: 2023-09-22T13:39:31.625 INFO:tasks.workunit.client.0.smithi069.stdout: client.5127 expiration=0.000000 addr=v1:172.21.15.69:0/2059596346 cookie= 2023-09-22T13:39:32.600 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] ClsLock.TestMeta (3016 ms) 2023-09-22T13:39:32.600 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] ClsLock.TestCookie 2023-09-22T13:39:34.632 INFO:tasks.workunit.client.0.smithi069.stdout:lock: mylock 2023-09-22T13:39:34.679 INFO:tasks.workunit.client.0.smithi069.stdout: lock_type: exclusive 2023-09-22T13:39:34.679 INFO:tasks.workunit.client.0.smithi069.stdout: tag: 2023-09-22T13:39:34.679 INFO:tasks.workunit.client.0.smithi069.stdout: lockers: 2023-09-22T13:39:34.680 INFO:tasks.workunit.client.0.smithi069.stdout:lock: mylock 2023-09-22T13:39:34.680 INFO:tasks.workunit.client.0.smithi069.stdout: lock_type: shared 2023-09-22T13:39:34.680 INFO:tasks.workunit.client.0.smithi069.stdout: tag: 2023-09-22T13:39:34.680 INFO:tasks.workunit.client.0.smithi069.stdout: lockers: 2023-09-22T13:39:34.682 INFO:tasks.workunit.client.0.smithi069.stdout: client.4937 expiration=0.000000 addr=v1:172.21.15.69:0/3633341497 cookie= 2023-09-22T13:39:34.682 INFO:tasks.workunit.client.0.smithi069.stdout: client.4937 expiration=0.000000 addr=v1:172.21.15.69:0/3633341497 cookie=new cookie 2023-09-22T13:39:35.632 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] ClsLock.TestCookie (3031 ms) 2023-09-22T13:39:35.632 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] ClsLock.TestMultipleLocks 2023-09-22T13:39:38.653 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] ClsLock.TestMultipleLocks (3021 ms) 2023-09-22T13:39:38.654 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] ClsLock.TestLockDuration 2023-09-22T13:39:46.695 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] ClsLock.TestLockDuration (8041 ms) 2023-09-22T13:39:46.696 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] ClsLock.TestAssertLocked 2023-09-22T13:39:49.729 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] ClsLock.TestAssertLocked (3036 ms) 2023-09-22T13:39:49.730 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] ClsLock.TestSetCookie 2023-09-22T13:39:52.755 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] ClsLock.TestSetCookie (3025 ms) 2023-09-22T13:39:52.756 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] ClsLock.TestRenew 2023-09-22T13:40:13.582 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] ClsLock.TestRenew (20828 ms) 2023-09-22T13:40:13.583 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralBasic 2023-09-22T13:40:20.617 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] ClsLock.TestExclusiveEphemeralBasic (7035 ms) 2023-09-22T13:40:20.618 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealEphemeral 2023-09-22T13:40:28.676 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealEphemeral (8059 ms) 2023-09-22T13:40:28.676 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealExclusive 2023-09-22T13:40:36.720 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealExclusive (8042 ms) 2023-09-22T13:40:36.720 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 11 tests from ClsLock (69667 ms total) 2023-09-22T13:40:36.720 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:40:36.720 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:40:36.720 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 11 tests from 1 test suite ran. (69667 ms total) 2023-09-22T13:40:36.720 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 11 tests. 2023-09-22T13:40:36.721 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:40:36.721 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:40:36.792 INFO:tasks.workunit:Running workunits matching cls/test_cls_log.sh on client.0... 2023-09-22T13:40:36.793 INFO:tasks.workunit:Running workunit cls/test_cls_log.sh... 2023-09-22T13:40:36.794 DEBUG:teuthology.orchestra.run.smithi069:workunit test cls/test_cls_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_log.sh 2023-09-22T13:40:36.865 INFO:tasks.workunit.client.0.smithi069.stdout:Running main() from gmock_main.cc 2023-09-22T13:40:36.865 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 4 tests from 1 test suite. 2023-09-22T13:40:36.865 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:40:36.866 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 4 tests from cls_log 2023-09-22T13:40:36.866 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_log.test_log_add_same_time 2023-09-22T13:40:39.746 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_log.test_log_add_same_time (2881 ms) 2023-09-22T13:40:39.774 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_log.test_log_add_different_time 2023-09-22T13:40:42.760 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_log.test_log_add_different_time (3014 ms) 2023-09-22T13:40:42.760 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_log.trim_by_time 2023-09-22T13:40:45.785 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_log.trim_by_time (3025 ms) 2023-09-22T13:40:45.785 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_log.trim_by_marker 2023-09-22T13:40:48.800 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_log.trim_by_marker (3014 ms) 2023-09-22T13:40:48.800 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 4 tests from cls_log (11935 ms total) 2023-09-22T13:40:48.800 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:40:48.800 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:40:48.800 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 4 tests from 1 test suite ran. (11935 ms total) 2023-09-22T13:40:48.800 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 4 tests. 2023-09-22T13:40:48.802 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:40:48.802 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:40:48.874 INFO:tasks.workunit:Running workunits matching cls/test_cls_refcount.sh on client.0... 2023-09-22T13:40:48.875 INFO:tasks.workunit:Running workunit cls/test_cls_refcount.sh... 2023-09-22T13:40:48.875 DEBUG:teuthology.orchestra.run.smithi069:workunit test cls/test_cls_refcount.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_refcount.sh 2023-09-22T13:40:48.946 INFO:tasks.workunit.client.0.smithi069.stdout:Running main() from gmock_main.cc 2023-09-22T13:40:48.946 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 10 tests from 1 test suite. 2023-09-22T13:40:48.946 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:40:48.946 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 10 tests from cls_rgw 2023-09-22T13:40:48.947 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.test_implicit 2023-09-22T13:40:51.844 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.test_implicit (2898 ms) 2023-09-22T13:40:51.845 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.test_implicit_idempotent 2023-09-22T13:40:54.859 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.test_implicit_idempotent (3015 ms) 2023-09-22T13:40:54.859 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.test_put_snap 2023-09-22T13:41:03.908 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.test_put_snap (9049 ms) 2023-09-22T13:41:03.908 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.test_explicit 2023-09-22T13:41:06.941 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.test_explicit (3033 ms) 2023-09-22T13:41:06.941 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.set 2023-09-22T13:41:09.967 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.set (3026 ms) 2023-09-22T13:41:09.967 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.test_implicit_ec 2023-09-22T13:41:16.992 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.test_implicit_ec (7025 ms) 2023-09-22T13:41:16.992 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.test_implicit_idempotent_ec 2023-09-22T13:41:24.034 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.test_implicit_idempotent_ec (7042 ms) 2023-09-22T13:41:24.035 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.test_put_snap_ec 2023-09-22T13:41:36.174 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.test_put_snap_ec (12140 ms) 2023-09-22T13:41:36.174 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.test_explicit_ec 2023-09-22T13:41:43.194 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.test_explicit_ec (7019 ms) 2023-09-22T13:41:43.195 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.set_ec 2023-09-22T13:41:50.233 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.set_ec (7036 ms) 2023-09-22T13:41:50.233 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 10 tests from cls_rgw (61284 ms total) 2023-09-22T13:41:50.233 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:41:50.233 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:41:50.233 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 10 tests from 1 test suite ran. (61284 ms total) 2023-09-22T13:41:50.233 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 10 tests. 2023-09-22T13:41:50.233 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:41:50.234 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:41:50.305 INFO:tasks.workunit:Running workunits matching cls/test_cls_rgw.sh on client.0... 2023-09-22T13:41:50.306 INFO:tasks.workunit:Running workunit cls/test_cls_rgw.sh... 2023-09-22T13:41:50.307 DEBUG:teuthology.orchestra.run.smithi069:workunit test cls/test_cls_rgw.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rgw.sh 2023-09-22T13:41:50.380 INFO:tasks.workunit.client.0.smithi069.stdout:Running main() from gmock_main.cc 2023-09-22T13:41:50.380 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 16 tests from 1 test suite. 2023-09-22T13:41:50.380 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:41:50.380 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 16 tests from cls_rgw 2023-09-22T13:41:52.236 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.index_basic 2023-09-22T13:41:52.305 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.index_basic (69 ms) 2023-09-22T13:41:52.305 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.index_multiple_obj_writers 2023-09-22T13:41:52.346 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.index_multiple_obj_writers (41 ms) 2023-09-22T13:41:52.346 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.index_remove_object 2023-09-22T13:41:52.395 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.index_remove_object (49 ms) 2023-09-22T13:41:52.396 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.index_suggest 2023-09-22T13:41:54.035 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.index_suggest (1640 ms) 2023-09-22T13:41:54.035 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.index_suggest_complete 2023-09-22T13:41:54.041 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.index_suggest_complete (6 ms) 2023-09-22T13:41:54.041 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.index_list 2023-09-22T13:41:54.063 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.index_list (22 ms) 2023-09-22T13:41:54.063 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.index_list_delimited 2023-09-22T13:42:15.667 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.index_list_delimited (21604 ms) 2023-09-22T13:42:15.667 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.bi_list 2023-09-22T13:42:15.831 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.bi_list (164 ms) 2023-09-22T13:42:15.832 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.gc_set 2023-09-22T13:42:15.843 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.gc_set (12 ms) 2023-09-22T13:42:15.843 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.gc_list 2023-09-22T13:42:15.855 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.gc_list (12 ms) 2023-09-22T13:42:15.855 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.gc_defer 2023-09-22T13:42:23.570 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.gc_defer (7714 ms) 2023-09-22T13:42:23.570 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.usage_basic 2023-09-22T13:42:23.869 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.usage_basic (300 ms) 2023-09-22T13:42:23.870 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.usage_clear_no_obj 2023-09-22T13:42:23.870 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.usage_clear_no_obj (1 ms) 2023-09-22T13:42:23.871 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.usage_clear 2023-09-22T13:42:24.019 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.usage_clear (149 ms) 2023-09-22T13:42:24.019 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.bi_log_trim 2023-09-22T13:42:24.066 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.bi_log_trim (47 ms) 2023-09-22T13:42:24.066 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] cls_rgw.index_racing_removes 2023-09-22T13:42:24.080 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] cls_rgw.index_racing_removes (14 ms) 2023-09-22T13:42:24.684 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 16 tests from cls_rgw (31845 ms total) 2023-09-22T13:42:24.684 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:42:24.684 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:42:24.684 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 16 tests from 1 test suite ran. (34304 ms total) 2023-09-22T13:42:24.684 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 16 tests. 2023-09-22T13:42:24.686 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:42:24.686 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:42:24.754 INFO:tasks.workunit:Running workunits matching cls/test_cls_cmpomap.sh on client.0... 2023-09-22T13:42:24.755 INFO:tasks.workunit:Running workunit cls/test_cls_cmpomap.sh... 2023-09-22T13:42:24.756 DEBUG:teuthology.orchestra.run.smithi069:workunit test cls/test_cls_cmpomap.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cmpomap.sh 2023-09-22T13:42:24.825 INFO:tasks.workunit.client.0.smithi069.stdout:Running main() from gmock_main.cc 2023-09-22T13:42:24.825 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 31 tests from 1 test suite. 2023-09-22T13:42:24.825 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:42:26.704 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 31 tests from CmpOmap 2023-09-22T13:42:26.704 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str 2023-09-22T13:42:26.709 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str (5 ms) 2023-09-22T13:42:26.709 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str_default 2023-09-22T13:42:26.892 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str_default (182 ms) 2023-09-22T13:42:26.892 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64 2023-09-22T13:42:26.895 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64 (4 ms) 2023-09-22T13:42:26.895 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64_default 2023-09-22T13:42:26.899 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64_default (4 ms) 2023-09-22T13:42:26.899 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_str 2023-09-22T13:42:26.907 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_str (8 ms) 2023-09-22T13:42:26.907 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_u64 2023-09-22T13:42:26.914 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_u64 (7 ms) 2023-09-22T13:42:26.914 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_input 2023-09-22T13:42:26.917 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_input (3 ms) 2023-09-22T13:42:26.918 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_default 2023-09-22T13:42:26.921 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_default (4 ms) 2023-09-22T13:42:26.921 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_u64_empty_default 2023-09-22T13:42:26.923 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_u64_empty_default (2 ms) 2023-09-22T13:42:26.923 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_value 2023-09-22T13:42:26.930 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_value (7 ms) 2023-09-22T13:42:26.930 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_at_max_keys 2023-09-22T13:42:26.931 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_at_max_keys (1 ms) 2023-09-22T13:42:26.931 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_vals_over_max_keys 2023-09-22T13:42:26.932 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_vals_over_max_keys (1 ms) 2023-09-22T13:42:26.932 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str 2023-09-22T13:42:26.938 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str (6 ms) 2023-09-22T13:42:26.938 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str_default 2023-09-22T13:42:26.946 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str_default (8 ms) 2023-09-22T13:42:26.946 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64 2023-09-22T13:42:26.953 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64 (7 ms) 2023-09-22T13:42:26.953 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64_default 2023-09-22T13:42:26.960 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64_default (7 ms) 2023-09-22T13:42:26.961 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_set_vals_str 2023-09-22T13:42:26.969 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_set_vals_str (9 ms) 2023-09-22T13:42:26.969 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64 2023-09-22T13:42:26.980 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_set_vals_u64 (11 ms) 2023-09-22T13:42:26.980 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_einval 2023-09-22T13:42:26.983 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_einval (3 ms) 2023-09-22T13:42:26.983 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_eio 2023-09-22T13:42:26.986 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_eio (3 ms) 2023-09-22T13:42:26.986 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_set_vals_at_max_keys 2023-09-22T13:42:26.987 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_set_vals_at_max_keys (1 ms) 2023-09-22T13:42:26.987 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_set_vals_over_max_keys 2023-09-22T13:42:26.987 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_set_vals_over_max_keys (1 ms) 2023-09-22T13:42:26.988 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_str 2023-09-22T13:42:26.994 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_str (6 ms) 2023-09-22T13:42:26.994 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_u64 2023-09-22T13:42:27.000 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_u64 (6 ms) 2023-09-22T13:42:27.000 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_rm_keys_str 2023-09-22T13:42:27.007 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_rm_keys_str (8 ms) 2023-09-22T13:42:27.008 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64 2023-09-22T13:42:27.017 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64 (9 ms) 2023-09-22T13:42:27.017 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_einval 2023-09-22T13:42:27.019 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_einval (2 ms) 2023-09-22T13:42:27.019 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_eio 2023-09-22T13:42:27.021 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_eio (3 ms) 2023-09-22T13:42:27.022 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_rm_keys_at_max_keys 2023-09-22T13:42:27.022 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_rm_keys_at_max_keys (1 ms) 2023-09-22T13:42:27.023 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_rm_keys_over_max_keys 2023-09-22T13:42:27.023 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_rm_keys_over_max_keys (0 ms) 2023-09-22T13:42:27.023 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_empty 2023-09-22T13:42:27.031 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_empty (8 ms) 2023-09-22T13:42:27.031 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 31 tests from CmpOmap (327 ms total) 2023-09-22T13:42:27.032 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:42:27.032 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:42:27.708 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 31 tests from 1 test suite ran. (2883 ms total) 2023-09-22T13:42:27.708 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 31 tests. 2023-09-22T13:42:27.710 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:42:27.710 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:42:27.781 INFO:tasks.workunit:Running workunits matching cls/test_cls_2pc_queue.sh on client.0... 2023-09-22T13:42:27.783 INFO:tasks.workunit:Running workunit cls/test_cls_2pc_queue.sh... 2023-09-22T13:42:27.783 DEBUG:teuthology.orchestra.run.smithi069:workunit test cls/test_cls_2pc_queue.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh 2023-09-22T13:42:27.855 INFO:tasks.workunit.client.0.smithi069.stdout:Running main() from gmock_main.cc 2023-09-22T13:42:27.855 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 22 tests from 1 test suite. 2023-09-22T13:42:27.855 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:42:27.855 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 22 tests from TestCls2PCQueue 2023-09-22T13:42:27.855 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.GetCapacity 2023-09-22T13:42:30.727 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.GetCapacity (2872 ms) 2023-09-22T13:42:30.728 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.AsyncGetCapacity 2023-09-22T13:42:33.750 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.AsyncGetCapacity (3023 ms) 2023-09-22T13:42:33.750 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.Reserve 2023-09-22T13:42:36.757 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.Reserve (3006 ms) 2023-09-22T13:42:36.757 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.AsyncReserve 2023-09-22T13:42:39.778 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.AsyncReserve (3021 ms) 2023-09-22T13:42:39.778 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.Commit 2023-09-22T13:42:42.794 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.Commit (3017 ms) 2023-09-22T13:42:42.795 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.Abort 2023-09-22T13:42:45.813 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.Abort (3019 ms) 2023-09-22T13:42:45.813 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.ReserveError 2023-09-22T13:42:48.851 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.ReserveError (3037 ms) 2023-09-22T13:42:48.851 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.CommitError 2023-09-22T13:42:51.869 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.CommitError (3018 ms) 2023-09-22T13:42:51.870 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.AbortError 2023-09-22T13:42:54.890 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.AbortError (3022 ms) 2023-09-22T13:42:54.891 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.MultiReserve 2023-09-22T13:42:57.920 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.MultiReserve (3030 ms) 2023-09-22T13:42:57.921 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.MultiCommit 2023-09-22T13:43:00.945 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.MultiCommit (3025 ms) 2023-09-22T13:43:00.945 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.MultiAbort 2023-09-22T13:43:03.958 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.MultiAbort (3012 ms) 2023-09-22T13:43:03.958 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.ReserveCommit 2023-09-22T13:43:06.995 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.ReserveCommit (3038 ms) 2023-09-22T13:43:07.095 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.ReserveAbort 2023-09-22T13:43:10.020 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.ReserveAbort (3025 ms) 2023-09-22T13:43:10.020 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.ManualCleanup 2023-09-22T13:43:13.036 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.ManualCleanup (3016 ms) 2023-09-22T13:43:13.036 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.Cleanup 2023-09-22T13:43:16.050 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.Cleanup (3013 ms) 2023-09-22T13:43:16.050 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.MultiProducer 2023-09-22T13:43:34.600 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.MultiProducer (18549 ms) 2023-09-22T13:43:34.601 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.AsyncConsumer 2023-09-22T13:43:37.622 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.AsyncConsumer (3024 ms) 2023-09-22T13:43:37.622 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.MultiProducerConsumer 2023-09-22T13:43:58.597 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.MultiProducerConsumer (20974 ms) 2023-09-22T13:43:58.597 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.ReserveSpillover 2023-09-22T13:44:03.621 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.ReserveSpillover (5025 ms) 2023-09-22T13:44:03.621 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.CommitSpillover 2023-09-22T13:44:10.658 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.CommitSpillover (7037 ms) 2023-09-22T13:44:10.659 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestCls2PCQueue.AbortSpillover 2023-09-22T13:44:17.695 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestCls2PCQueue.AbortSpillover (7036 ms) 2023-09-22T13:44:17.695 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 22 tests from TestCls2PCQueue (109839 ms total) 2023-09-22T13:44:17.695 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:17.695 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:17.695 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 22 tests from 1 test suite ran. (109839 ms total) 2023-09-22T13:44:17.695 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 22 tests. 2023-09-22T13:44:17.697 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:44:17.698 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:44:17.769 INFO:tasks.workunit:Running workunits matching rgw/test_rgw_gc_log.sh on client.0... 2023-09-22T13:44:17.771 INFO:tasks.workunit:Running workunit rgw/test_rgw_gc_log.sh... 2023-09-22T13:44:17.771 DEBUG:teuthology.orchestra.run.smithi069:workunit test rgw/test_rgw_gc_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_gc_log.sh 2023-09-22T13:44:17.853 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:17.853 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:17.853 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 6 tests from 1 test suite. 2023-09-22T13:44:17.854 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:19.701 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 6 tests from rgw_gc_log 2023-09-22T13:44:19.701 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] rgw_gc_log.init_existing_queue 2023-09-22T13:44:19.718 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] rgw_gc_log.init_existing_queue (19 ms) 2023-09-22T13:44:19.718 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] rgw_gc_log.init_existing_omap 2023-09-22T13:44:19.722 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] rgw_gc_log.init_existing_omap (5 ms) 2023-09-22T13:44:19.723 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] rgw_gc_log.enqueue1_after_init 2023-09-22T13:44:19.725 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] rgw_gc_log.enqueue1_after_init (3 ms) 2023-09-22T13:44:19.726 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] rgw_gc_log.enqueue2_before_init 2023-09-22T13:44:19.727 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] rgw_gc_log.enqueue2_before_init (1 ms) 2023-09-22T13:44:19.727 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] rgw_gc_log.defer1_after_init 2023-09-22T13:44:19.730 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] rgw_gc_log.defer1_after_init (3 ms) 2023-09-22T13:44:19.730 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] rgw_gc_log.defer2_before_init 2023-09-22T13:44:19.731 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] rgw_gc_log.defer2_before_init (1 ms) 2023-09-22T13:44:19.731 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 6 tests from rgw_gc_log (32 ms total) 2023-09-22T13:44:19.731 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:19.731 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:20.699 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 6 tests from 1 test suite ran. (2846 ms total) 2023-09-22T13:44:20.699 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 6 tests. 2023-09-22T13:44:20.703 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:44:20.703 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:44:20.778 INFO:tasks.workunit:Running workunits matching rgw/test_rgw_obj.sh on client.0... 2023-09-22T13:44:20.779 INFO:tasks.workunit:Running workunit rgw/test_rgw_obj.sh... 2023-09-22T13:44:20.779 DEBUG:teuthology.orchestra.run.smithi069:workunit test rgw/test_rgw_obj.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_obj.sh 2023-09-22T13:44:20.861 INFO:tasks.workunit.client.0.smithi069.stdout:Running main() from gmock_main.cc 2023-09-22T13:44:20.861 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 4 tests from 1 test suite. 2023-09-22T13:44:20.861 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:20.861 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 4 tests from TestRGWObj 2023-09-22T13:44:20.861 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestRGWObj.underscore 2023-09-22T13:44:20.861 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.861 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns= i= 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns= i= 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns= i= 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout: "obj1": { 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:20.862 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:20.863 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: "obj2": { 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.864 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.865 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:20.865 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:20.865 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:20.865 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:20.865 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.865 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.865 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3": { 2023-09-22T13:44:20.865 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:20.865 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:20.865 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:20.865 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:20.866 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:20.866 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:20.866 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:20.866 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:20.866 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:20.866 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.866 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.866 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:20.866 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:20.866 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3-2": { 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:20.867 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.868 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns=ns i= 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns=ns i= 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns=ns i= 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout: "obj1": { 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:20.869 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.870 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.871 INFO:tasks.workunit.client.0.smithi069.stdout: "obj2": { 2023-09-22T13:44:20.871 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:20.871 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:20.871 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:20.871 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:20.871 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:20.871 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:20.871 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:20.871 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:20.871 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3": { 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:20.872 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:20.873 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:20.873 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:20.873 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:20.873 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:20.873 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:20.873 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.873 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.873 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:20.873 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:20.873 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:20.873 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:20.874 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.874 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.874 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3-2": { 2023-09-22T13:44:20.874 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:20.874 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:20.874 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:20.997 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:20.997 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:20.997 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:20.997 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:20.998 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:20.998 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:20.998 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.998 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:20.998 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:20.998 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:20.998 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:20.998 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:20.998 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.998 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:20.998 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:20.999 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:20.999 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:20.999 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns= i=v1 2023-09-22T13:44:20.999 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:20.999 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:20.999 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns= i=v1 2023-09-22T13:44:20.999 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:20.999 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:20.999 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns= i=v1 2023-09-22T13:44:21.000 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.000 INFO:tasks.workunit.client.0.smithi069.stdout: "obj1": { 2023-09-22T13:44:21.000 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.000 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.000 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.000 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.000 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.000 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.000 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.000 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.001 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.001 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.001 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.001 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.001 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:21.001 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.001 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.001 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.001 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.001 INFO:tasks.workunit.client.0.smithi069.stdout: "obj2": { 2023-09-22T13:44:21.001 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.002 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.002 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.002 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.002 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.002 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.002 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.002 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.002 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.002 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.002 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.002 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.003 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:21.003 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.003 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.003 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.003 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.003 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3": { 2023-09-22T13:44:21.003 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.003 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.003 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.003 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.003 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.004 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.004 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.004 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.004 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.004 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.004 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.004 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.004 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:21.005 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.005 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.005 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.005 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.005 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3-2": { 2023-09-22T13:44:21.005 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.005 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.005 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.006 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.006 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.006 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.006 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.006 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.006 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.006 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.006 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.006 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.007 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:21.007 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.007 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.007 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.007 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.007 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.007 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:21.007 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:21.007 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns=ns i=v1 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns=ns i=v1 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: _obj[v1] 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: _obj ns=ns i=v1 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout: "obj1": { 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.008 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.009 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.009 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.009 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.009 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.009 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.009 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.009 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.009 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.009 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:21.009 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.009 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: "obj2": { 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.010 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3": { 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.011 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.012 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.012 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.012 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.012 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.012 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.012 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.012 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.012 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.012 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.012 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:21.012 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.013 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.013 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.013 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.013 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3-2": { 2023-09-22T13:44:21.013 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.013 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.013 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.013 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.014 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.014 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.014 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.014 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.014 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.014 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.014 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.014 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.014 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_obj", 2023-09-22T13:44:21.014 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestRGWObj.underscore (0 ms) 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestRGWObj.no_underscore 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns= i= 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.015 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns= i= 2023-09-22T13:44:21.016 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.016 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.016 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns= i= 2023-09-22T13:44:21.016 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.016 INFO:tasks.workunit.client.0.smithi069.stdout: "obj1": { 2023-09-22T13:44:21.016 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.016 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.016 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.016 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.016 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.016 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.017 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.017 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.017 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.017 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.017 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.017 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.017 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.017 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.017 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.017 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.017 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.018 INFO:tasks.workunit.client.0.smithi069.stdout: "obj2": { 2023-09-22T13:44:21.018 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.018 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.018 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.018 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.018 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.018 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.018 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.018 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.018 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.018 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3": { 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.019 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.020 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.021 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.021 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3-2": { 2023-09-22T13:44:21.021 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.021 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.021 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.021 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.021 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.021 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.021 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.021 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.021 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.022 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns=ns i= 2023-09-22T13:44:21.023 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.023 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.023 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns=ns i= 2023-09-22T13:44:21.023 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.023 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj 2023-09-22T13:44:21.023 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns=ns i= 2023-09-22T13:44:21.023 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.023 INFO:tasks.workunit.client.0.smithi069.stdout: "obj1": { 2023-09-22T13:44:21.023 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.023 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.023 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.024 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: "obj2": { 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.025 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.026 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.026 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.026 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.026 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.026 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.026 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.026 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.026 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.026 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3": { 2023-09-22T13:44:21.026 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.026 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.027 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3-2": { 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.028 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.029 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns= i=v1 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns= i=v1 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns= i=v1 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout: "obj1": { 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.030 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.031 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: "obj2": { 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.032 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3": { 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.033 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3-2": { 2023-09-22T13:44:21.034 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.035 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns=ns i=v1 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns=ns i=v1 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.036 INFO:tasks.workunit.client.0.smithi069.stdout:parsed: obj[v1] 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout:stripped: obj ns=ns i=v1 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout: "obj1": { 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.037 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: "obj2": { 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.038 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.039 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3": { 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.040 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: "obj3-2": { 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.041 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "obj", 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "v1", 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.042 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestRGWObj.no_underscore (0 ms) 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestRGWObj.obj_to_raw 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.043 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.044 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.045 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker_myobj", 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.046 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.047 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.048 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.049 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker_myobj", 2023-09-22T13:44:21.050 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.051 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.052 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.053 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.054 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.055 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.056 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.057 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.058 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.059 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.060 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.061 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.062 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.063 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.064 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.065 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.066 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.067 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.068 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.069 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.070 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:44:21.070 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.070 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.070 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.070 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.070 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.070 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.070 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.070 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.070 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.070 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.071 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.072 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.073 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.074 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.074 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.074 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.074 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.074 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.074 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.074 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.074 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.074 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.074 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.074 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.075 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.076 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.077 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.078 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.079 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:44:21.080 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.080 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.080 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.080 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.080 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.080 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.080 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.080 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.080 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.080 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.080 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.081 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.082 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.082 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.082 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.082 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.082 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.082 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.082 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.082 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.082 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.082 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.083 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.084 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.084 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.084 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.084 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.084 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.084 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.084 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.084 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.084 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.084 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.084 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker___myobj", 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj" 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.085 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.086 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.086 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.086 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.086 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.086 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.086 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.086 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.086 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.086 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.086 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.086 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.087 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.087 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.087 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.087 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.087 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.087 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.087 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.087 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.087 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.087 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.087 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.088 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.089 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.090 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.090 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.090 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.090 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.090 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.090 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker___myobj", 2023-09-22T13:44:21.090 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj" 2023-09-22T13:44:21.090 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.090 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.090 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.091 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.092 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.093 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.094 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.094 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.094 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.094 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.094 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.094 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.094 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.094 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.094 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.094 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.094 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.095 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.095 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.095 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.095 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.095 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.095 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.095 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.095 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.095 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:44:21.095 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.095 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.096 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.097 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.098 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.098 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.098 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.098 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.098 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.098 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.098 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.098 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.098 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.098 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.098 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.099 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.100 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.101 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.102 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.103 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.103 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.103 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.103 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.103 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.103 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.103 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.103 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.103 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.103 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.103 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.104 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.104 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.104 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.104 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.104 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.104 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.104 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.104 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.104 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.104 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.104 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.105 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.105 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.105 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.105 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.105 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.105 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.105 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.105 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.105 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.105 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.105 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.106 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:44:21.106 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj" 2023-09-22T13:44:21.106 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.106 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.106 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.106 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.106 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.106 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.106 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.106 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.106 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.107 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.108 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.109 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.110 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj" 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.111 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.112 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.113 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.114 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.115 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.116 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.117 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.118 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.119 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.120 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.121 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.122 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.123 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.124 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker___myobj_", 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj_" 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.125 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.126 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.127 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.128 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.129 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker___myobj_", 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj_" 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.130 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.131 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.132 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.133 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:44:21.134 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.135 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.136 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.137 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.138 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.139 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.140 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.141 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.142 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.143 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj_" 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.144 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.145 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.146 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.147 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.148 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj_" 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.149 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.150 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.151 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.152 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": "default-placement.data", 2023-09-22T13:44:21.153 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "test", 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.154 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": "", 2023-09-22T13:44:21.155 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.155 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": "" 2023-09-22T13:44:21.155 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.155 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.155 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.155 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.155 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.157 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.157 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.157 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.157 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.157 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.157 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.157 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.157 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout: "obj": { 2023-09-22T13:44:21.158 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.159 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.160 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.161 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestRGWObj.obj_to_raw (0 ms) 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] TestRGWObj.old_to_raw 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.162 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker_myobj", 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.163 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker_myobj 2023-09-22T13:44:21.164 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker_myobj", 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.165 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.166 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker_myobj 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.167 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.168 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker__ns_myobj 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.169 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.170 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker__ns_myobj 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.171 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.172 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker__:inst_myobj 2023-09-22T13:44:21.173 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.174 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.175 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.175 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.175 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.175 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.175 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.175 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.175 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.175 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.175 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.175 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.175 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker__:inst_myobj 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.176 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.177 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker__ns:inst_myobj 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.178 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.179 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "myobj", 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker__ns:inst_myobj 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.180 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker___myobj", 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj" 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.181 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker___myobj 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.182 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker___myobj", 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj" 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.183 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.184 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker___myobj 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.185 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.186 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker__ns__myobj 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.187 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.188 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker__ns__myobj 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj" 2023-09-22T13:44:21.189 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.190 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker__:inst__myobj 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj" 2023-09-22T13:44:21.191 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.192 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker__:inst__myobj 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.193 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.194 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.195 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.195 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.195 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.195 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.195 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.195 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.195 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.195 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.195 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.195 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.195 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker__ns:inst__myobj 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.196 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.197 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj", 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker__ns:inst__myobj 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker___myobj_", 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj_" 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.198 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.199 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.199 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.199 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.199 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.199 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.199 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.199 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.199 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.199 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.199 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.199 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker___myobj_ 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.200 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker___myobj_", 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj_" 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.201 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker___myobj_ 2023-09-22T13:44:21.202 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.203 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.204 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker__ns__myobj_ 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.205 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.206 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "", 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker__ns__myobj_ 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj_" 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.207 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.208 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker__:inst__myobj_ 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:44:21.209 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "marker__myobj_" 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.210 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "" 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker__:inst__myobj_ 2023-09-22T13:44:21.211 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj": { 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj": { 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.212 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.213 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout:raw=.data-pool:marker__ns:inst__myobj_ 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout: "raw_obj2": { 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout: "pool": ".data-pool", 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout: "loc": "" 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout: "new_obj2": { 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket": { 2023-09-22T13:44:21.214 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "ebtest", 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: "marker": "marker", 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_id": "bucket-id", 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: "tenant": "", 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: "explicit_placement": { 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: "data_pool": ".data-pool", 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: "data_extra_pool": "", 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: "index_pool": ".index-pool" 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: "key": { 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: "name": "_myobj_", 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: "instance": "inst", 2023-09-22T13:44:21.215 INFO:tasks.workunit.client.0.smithi069.stdout: "ns": "ns" 2023-09-22T13:44:21.216 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.216 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:21.216 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:21.216 INFO:tasks.workunit.client.0.smithi069.stdout:raw2=.data-pool:marker__ns:inst__myobj_ 2023-09-22T13:44:21.216 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] TestRGWObj.old_to_raw (1 ms) 2023-09-22T13:44:21.216 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 4 tests from TestRGWObj (1 ms total) 2023-09-22T13:44:21.216 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:21.216 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:21.216 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 4 tests from 1 test suite ran. (1 ms total) 2023-09-22T13:44:21.216 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 4 tests. 2023-09-22T13:44:21.217 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:44:21.217 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:44:21.249 INFO:tasks.workunit:Running workunits matching rgw/test_rgw_throttle.sh on client.0... 2023-09-22T13:44:21.250 INFO:tasks.workunit:Running workunit rgw/test_rgw_throttle.sh... 2023-09-22T13:44:21.250 DEBUG:teuthology.orchestra.run.smithi069:workunit test rgw/test_rgw_throttle.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_throttle.sh 2023-09-22T13:44:21.336 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:21.336 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:21.337 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 5 tests from 1 test suite. 2023-09-22T13:44:21.337 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:22.741 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 5 tests from Aio_Throttle 2023-09-22T13:44:22.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] Aio_Throttle.NoThrottleUpToMax 2023-09-22T13:44:22.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] Aio_Throttle.NoThrottleUpToMax (0 ms) 2023-09-22T13:44:22.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] Aio_Throttle.CostOverWindow 2023-09-22T13:44:22.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] Aio_Throttle.CostOverWindow (0 ms) 2023-09-22T13:44:22.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] Aio_Throttle.ThrottleOverMax 2023-09-22T13:44:22.829 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] Aio_Throttle.ThrottleOverMax (88 ms) 2023-09-22T13:44:22.829 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] Aio_Throttle.YieldCostOverWindow 2023-09-22T13:44:22.830 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] Aio_Throttle.YieldCostOverWindow (0 ms) 2023-09-22T13:44:22.830 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] Aio_Throttle.YieldingThrottleOverMax 2023-09-22T13:44:22.917 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] Aio_Throttle.YieldingThrottleOverMax (88 ms) 2023-09-22T13:44:22.918 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 5 tests from Aio_Throttle (176 ms total) 2023-09-22T13:44:22.918 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:22.918 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:22.918 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 5 tests from 1 test suite ran. (1581 ms total) 2023-09-22T13:44:22.918 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 5 tests. 2023-09-22T13:44:22.921 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:44:22.921 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:44:22.949 INFO:tasks.workunit:Running workunits matching rgw/test_librgw_file.sh on client.0... 2023-09-22T13:44:22.950 INFO:tasks.workunit:Running workunit rgw/test_librgw_file.sh... 2023-09-22T13:44:22.950 DEBUG:teuthology.orchestra.run.smithi069:workunit test rgw/test_librgw_file.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_librgw_file.sh 2023-09-22T13:44:23.045 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:23.045 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:23.054 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.052+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.056 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.054+0000 7f19d7c4b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:23.056 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.054+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.056 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.055+0000 7f19d7c4b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:23.056 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.055+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.056 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.055+0000 7f19d7c4b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:23.057 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.055+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.057 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.056+0000 7f19d7c4b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:23.269 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.267+0000 7f19d7c4b500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:44:23.269 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.267+0000 7f19d7c4b500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:44:23.269 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.267+0000 7f19d7c4b500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:44:23.269 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.267+0000 7f19d7c4b500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:44:23.269 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.267+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.269 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.267+0000 7f19d7c4b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:23.269 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.267+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.271 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.267+0000 7f19d7c4b500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:44:23.271 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.267+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.271 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.268+0000 7f19d7c4b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:23.271 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.268+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.271 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.268+0000 7f19d7c4b500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.268+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.268+0000 7f19d7c4b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.268+0000 7f19d7c4b500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.268+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.268+0000 7f19d7c4b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.269+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.269+0000 7f19d7c4b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.269+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.269+0000 7f19d7c4b500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.269+0000 7f19d7c4b500 20 zonegroup default 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.269+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.269+0000 7f19d7c4b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:23.273 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.269+0000 7f19d7c4b500 10 Cannot find current period zone using local zone 2023-09-22T13:44:23.273 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.269+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.273 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.269+0000 7f19d7c4b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:23.273 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.269+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.274 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.270+0000 7f19d7c4b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:23.274 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.270+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.274 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.270+0000 7f19d7c4b500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:44:23.274 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.270+0000 7f19d7c4b500 20 zone default found 2023-09-22T13:44:23.274 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.270+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.274 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.271+0000 7f19d7c4b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:23.274 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.271+0000 7f19d7c4b500 20 rados->read ofs=0 len=0 2023-09-22T13:44:23.274 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.271+0000 7f19d7c4b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:23.274 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.271+0000 7f19d7c4b500 20 started sync module instance, tier type = 2023-09-22T13:44:23.274 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.271+0000 7f19d7c4b500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:44:23.295 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.294+0000 7f19d7c4b500 20 add_watcher() i=0 2023-09-22T13:44:23.296 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.294+0000 7f19d7c4b500 20 add_watcher() i=1 2023-09-22T13:44:23.296 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.294+0000 7f19d7c4b500 20 add_watcher() i=2 2023-09-22T13:44:23.296 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.294+0000 7f19d7c4b500 20 add_watcher() i=3 2023-09-22T13:44:23.296 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.294+0000 7f19d7c4b500 20 add_watcher() i=4 2023-09-22T13:44:23.296 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.294+0000 7f19d7c4b500 20 add_watcher() i=5 2023-09-22T13:44:23.296 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.294+0000 7f19d7c4b500 20 add_watcher() i=6 2023-09-22T13:44:23.298 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.296+0000 7f19d7c4b500 20 add_watcher() i=7 2023-09-22T13:44:23.298 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.296+0000 7f19d7c4b500 2 all 8 watchers are set, enabling cache 2023-09-22T13:44:23.298 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.297+0000 7f19c392e700 0 -- 172.21.15.69:0/58633282 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x55f0abec5fe0 msgr2=0x55f0abec83d0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:44:23.300 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.298+0000 7f190ffff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:44:23.300 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.299+0000 7f19d7c4b500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:44:23.300 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.299+0000 7f19d7c4b500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:44:23.305 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.303+0000 7f19c292c700 0 -- 172.21.15.69:0/667442227 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x55f0abf18ed0 msgr2=0x55f0abf00de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:44:23.307 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.306+0000 7f19d7c4b500 5 note: GC not initialized 2023-09-22T13:44:23.309 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f1906ffd700 20 reqs_thread_entry: start 2023-09-22T13:44:23.309 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f1904ff9700 20 reqs_thread_entry: start 2023-09-22T13:44:23.309 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f19d7c4b500 20 init_complete bucket index max shards: 11 2023-09-22T13:44:23.309 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f19d7c4b500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:44:23.309 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f18f2ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:44:53 2023 2023-09-22T13:44:23.310 INFO:tasks.workunit.client.0.smithi069.stderr: 2023-09-22T13:44:23.310 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f18f27fc700 20 reqs_thread_entry: start 2023-09-22T13:44:23.310 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f19d7c4b500 20 get_system_obj_state: rctx=0x7ffe600c12b8 obj=default.rgw.meta:users.uid:ceph-test-librgw-file state=0x55f0abdd72b0 s->prefetch_data=0 2023-09-22T13:44:23.310 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f19d7c4b500 10 cache get: name=default.rgw.meta+users.uid+ceph-test-librgw-file : miss 2023-09-22T13:44:23.310 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f19d7c4b500 10 cache put: name=default.rgw.meta+users.uid+ceph-test-librgw-file info.flags=0x0 2023-09-22T13:44:23.310 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f19d7c4b500 10 adding default.rgw.meta+users.uid+ceph-test-librgw-file to cache LRU end 2023-09-22T13:44:23.310 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f19d7c4b500 20 get_system_obj_state: rctx=0x7ffe600c12c8 obj=default.rgw.meta:users.email:librgw@example.com state=0x55f0abdd72b0 s->prefetch_data=0 2023-09-22T13:44:23.310 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.307+0000 7f19d7c4b500 10 cache get: name=default.rgw.meta+users.email+librgw@example.com : miss 2023-09-22T13:44:23.311 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.308+0000 7f19d7c4b500 10 cache put: name=default.rgw.meta+users.email+librgw@example.com info.flags=0x0 2023-09-22T13:44:23.311 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.308+0000 7f19d7c4b500 10 adding default.rgw.meta+users.email+librgw@example.com to cache LRU end 2023-09-22T13:44:23.311 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.308+0000 7f19d7c4b500 20 get_system_obj_state: rctx=0x7ffe600c12c8 obj=default.rgw.meta:users.keys:WnQCoo7LDftV9O4OLz0h7LpQcxs= state=0x55f0abdd72b0 s->prefetch_data=0 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.308+0000 7f19d7c4b500 10 cache get: name=default.rgw.meta+users.keys+WnQCoo7LDftV9O4OLz0h7LpQcxs= : miss 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.308+0000 7f19d7c4b500 10 cache put: name=default.rgw.meta+users.keys+WnQCoo7LDftV9O4OLz0h7LpQcxs= info.flags=0x0 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.308+0000 7f19d7c4b500 10 adding default.rgw.meta+users.keys+WnQCoo7LDftV9O4OLz0h7LpQcxs= to cache LRU end 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.308+0000 7f19d7c4b500 20 get_system_obj_state: rctx=0x7ffe600c0bc8 obj=default.rgw.meta:users.keys:WnQCoo7LDftV9O4OLz0h7LpQcxs= state=0x55f0abdd72b0 s->prefetch_data=0 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.308+0000 7f19d7c4b500 10 cache get: name=default.rgw.meta+users.keys+WnQCoo7LDftV9O4OLz0h7LpQcxs= : hit (negative entry) 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.308+0000 7f19d7c4b500 20 get_system_obj_state: rctx=0x7ffe600c0f08 obj=default.rgw.meta:users.keys:WnQCoo7LDftV9O4OLz0h7LpQcxs= state=0x55f0abdd72b0 s->prefetch_data=0 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.308+0000 7f19d7c4b500 10 cache get: name=default.rgw.meta+users.keys+WnQCoo7LDftV9O4OLz0h7LpQcxs= : hit (negative entry) 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.309+0000 7f19d7c4b500 10 cache put: name=default.rgw.meta+users.uid+ceph-test-librgw-file info.flags=0x17 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.309+0000 7f19d7c4b500 10 moving default.rgw.meta+users.uid+ceph-test-librgw-file to cache LRU end 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.309+0000 7f19d7c4b500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.uid:ceph-test-librgw-file, ofs0, ns] 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.310+0000 7f19c1129700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1000727379969 cookie 94492164969584 notifier 5466 bl.length()=663 2023-09-22T13:44:23.312 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.310+0000 7f19c1129700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+ceph-test-librgw-file info.flags=0x17 2023-09-22T13:44:23.313 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.310+0000 7f19c1129700 10 rgw watcher librados: moving default.rgw.meta+users.uid+ceph-test-librgw-file to cache LRU end 2023-09-22T13:44:23.313 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.311+0000 7f19d7c4b500 10 cache put: name=default.rgw.meta+users.email+librgw@example.com info.flags=0x7 2023-09-22T13:44:23.313 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.311+0000 7f19d7c4b500 10 moving default.rgw.meta+users.email+librgw@example.com to cache LRU end 2023-09-22T13:44:23.313 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.311+0000 7f19d7c4b500 10 distributing notification oid=default.rgw.control:notify.1 cni=[op: 0, obj: default.rgw.meta:users.email:librgw@example.com, ofs0, ns] 2023-09-22T13:44:23.314 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.312+0000 7f19c0928700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1000727379968 cookie 94492164974320 notifier 5466 bl.length()=194 2023-09-22T13:44:23.314 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.312+0000 7f19c0928700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.email+librgw@example.com info.flags=0x7 2023-09-22T13:44:23.314 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.312+0000 7f19c0928700 10 rgw watcher librados: moving default.rgw.meta+users.email+librgw@example.com to cache LRU end 2023-09-22T13:44:23.315 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.313+0000 7f19d7c4b500 10 cache put: name=default.rgw.meta+users.keys+WnQCoo7LDftV9O4OLz0h7LpQcxs= info.flags=0x7 2023-09-22T13:44:23.315 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.313+0000 7f19d7c4b500 10 moving default.rgw.meta+users.keys+WnQCoo7LDftV9O4OLz0h7LpQcxs= to cache LRU end 2023-09-22T13:44:23.316 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.314+0000 7f19d7c4b500 10 distributing notification oid=default.rgw.control:notify.4 cni=[op: 0, obj: default.rgw.meta:users.keys:WnQCoo7LDftV9O4OLz0h7LpQcxs=, ofs0, ns] 2023-09-22T13:44:23.316 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.314+0000 7f19c1129700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1000727379969 cookie 94492164985376 notifier 5466 bl.length()=203 2023-09-22T13:44:23.316 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.314+0000 7f19c1129700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+WnQCoo7LDftV9O4OLz0h7LpQcxs= info.flags=0x7 2023-09-22T13:44:23.316 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.314+0000 7f19c1129700 10 rgw watcher librados: moving default.rgw.meta+users.keys+WnQCoo7LDftV9O4OLz0h7LpQcxs= to cache LRU end 2023-09-22T13:44:23.316 INFO:tasks.workunit.client.0.smithi069.stdout:{ 2023-09-22T13:44:23.316 INFO:tasks.workunit.client.0.smithi069.stdout: "user_id": "ceph-test-librgw-file", 2023-09-22T13:44:23.316 INFO:tasks.workunit.client.0.smithi069.stdout: "display_name": "librgw test user", 2023-09-22T13:44:23.316 INFO:tasks.workunit.client.0.smithi069.stdout: "email": "librgw@example.com", 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: "suspended": 0, 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: "max_buckets": 1000, 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: "subusers": [], 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: "keys": [ 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: { 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: "user": "ceph-test-librgw-file", 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: "access_key": "WnQCoo7LDftV9O4OLz0h7LpQcxs=", 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: "secret_key": "fZyTWQzYSEeMgzEIaIYFIHnzfmt8CI47x55uHpxVNhNvU56Ok0fgEw==" 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: } 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: ], 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: "swift_keys": [], 2023-09-22T13:44:23.317 INFO:tasks.workunit.client.0.smithi069.stdout: "caps": [], 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "op_mask": "read, write, delete", 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "default_placement": "", 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "default_storage_class": "", 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "placement_tags": [], 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "bucket_quota": { 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "enabled": false, 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "check_on_raw": false, 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "max_size": -1, 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "max_size_kb": 0, 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "max_objects": -1 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "user_quota": { 2023-09-22T13:44:23.318 INFO:tasks.workunit.client.0.smithi069.stdout: "enabled": false, 2023-09-22T13:44:23.319 INFO:tasks.workunit.client.0.smithi069.stdout: "check_on_raw": false, 2023-09-22T13:44:23.319 INFO:tasks.workunit.client.0.smithi069.stdout: "max_size": -1, 2023-09-22T13:44:23.319 INFO:tasks.workunit.client.0.smithi069.stdout: "max_size_kb": 0, 2023-09-22T13:44:23.319 INFO:tasks.workunit.client.0.smithi069.stdout: "max_objects": -1 2023-09-22T13:44:23.319 INFO:tasks.workunit.client.0.smithi069.stdout: }, 2023-09-22T13:44:23.319 INFO:tasks.workunit.client.0.smithi069.stdout: "temp_url_keys": [], 2023-09-22T13:44:23.319 INFO:tasks.workunit.client.0.smithi069.stdout: "type": "rgw", 2023-09-22T13:44:23.319 INFO:tasks.workunit.client.0.smithi069.stdout: "mfa_ids": [] 2023-09-22T13:44:23.319 INFO:tasks.workunit.client.0.smithi069.stdout:} 2023-09-22T13:44:23.319 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:23.320 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.316+0000 7f19d7c4b500 20 remove_watcher() i=0 2023-09-22T13:44:23.320 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.316+0000 7f19d7c4b500 2 removed watcher, disabling cache 2023-09-22T13:44:23.320 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.317+0000 7f19d7c4b500 20 remove_watcher() i=1 2023-09-22T13:44:23.320 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.318+0000 7f19d7c4b500 20 remove_watcher() i=2 2023-09-22T13:44:23.321 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.320+0000 7f19d7c4b500 20 remove_watcher() i=3 2023-09-22T13:44:23.323 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.321+0000 7f19d7c4b500 20 remove_watcher() i=4 2023-09-22T13:44:23.324 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.322+0000 7f19d7c4b500 20 remove_watcher() i=5 2023-09-22T13:44:23.325 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.323+0000 7f19d7c4b500 20 remove_watcher() i=6 2023-09-22T13:44:23.326 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.324+0000 7f19d7c4b500 20 remove_watcher() i=7 2023-09-22T13:44:23.327 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:23.325+0000 7f190ffff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:44:23.331 INFO:tasks.workunit.client.0.smithi069.stdout:phase 1.1 2023-09-22T13:44:23.348 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 24 tests from 1 test suite. 2023-09-22T13:44:23.348 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:23.348 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 24 tests from LibRGW 2023-09-22T13:44:23.348 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:44:23.359 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:23.359 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:23.505 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.INIT (157 ms) 2023-09-22T13:44:23.505 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT_NOROOT 2023-09-22T13:44:23.507 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT_NOROOT (2 ms) 2023-09-22T13:44:23.507 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:44:23.507 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT (0 ms) 2023-09-22T13:44:23.507 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SETUP_HIER1 2023-09-22T13:44:23.520 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/bar/baz/quux 2023-09-22T13:44:23.528 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/f1 2023-09-22T13:44:23.535 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/f2 2023-09-22T13:44:23.539 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/bar/f1 2023-09-22T13:44:23.543 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/bar/d1/ 2023-09-22T13:44:23.548 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/bar/baz/hungry 2023-09-22T13:44:23.555 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/bar/baz/hungry/ 2023-09-22T13:44:23.564 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/bar/baz/momma 2023-09-22T13:44:23.571 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/bar/baz/bear/ 2023-09-22T13:44:23.575 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/bar/baz/sasquatch 2023-09-22T13:44:23.581 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/bar/baz/sasquatch/ 2023-09-22T13:44:23.584 INFO:tasks.workunit.client.0.smithi069.stdout:creating: nfsroot:foo/bar/baz/frobozz 2023-09-22T13:44:23.590 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SETUP_HIER1 (83 ms) 2023-09-22T13:44:23.590 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SETUP_DIRS1 2023-09-22T13:44:23.741 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SETUP_DIRS1 (151 ms) 2023-09-22T13:44:23.741 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SETATTR 2023-09-22T13:44:23.741 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SETATTR (0 ms) 2023-09-22T13:44:23.741 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_CREATE_DIRS1 2023-09-22T13:44:23.775 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_CREATE_DIRS1 (34 ms) 2023-09-22T13:44:23.775 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_SETUP_RENAME1 2023-09-22T13:44:23.838 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_SETUP_RENAME1 (63 ms) 2023-09-22T13:44:23.838 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_INTRABUCKET_RENAME1 2023-09-22T13:44:23.838 INFO:tasks.workunit.client.0.smithi069.stdout:rename file rfile_0 to rfile_r0 (bucket brename0) 2023-09-22T13:44:23.845 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_INTRABUCKET_RENAME1 (7 ms) 2023-09-22T13:44:23.845 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_CROSSBUCKET_RENAME1 2023-09-22T13:44:23.845 INFO:tasks.workunit.client.0.smithi069.stdout:rename file rfile_1 (bucket brename0) to rfile_rhilldog (bucket brename1) 2023-09-22T13:44:23.852 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_CROSSBUCKET_RENAME1 (6 ms) 2023-09-22T13:44:23.852 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.GETATTR_DIRS1 2023-09-22T13:44:23.852 INFO:tasks.workunit.client.0.smithi069.stdout:scanning objects in /bdirs1/dir_0 2023-09-22T13:44:23.852 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_0/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:23.852 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_0/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:44:23.852 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_0/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:23.852 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_0/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:44:23.852 INFO:tasks.workunit.client.0.smithi069.stdout:scanning objects in /bdirs1/dir_1 2023-09-22T13:44:23.852 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_1/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:23.852 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_1/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:44:23.852 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_1/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:23.853 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_1/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:44:23.853 INFO:tasks.workunit.client.0.smithi069.stdout:scanning objects in /bdirs1/dir_2 2023-09-22T13:44:23.853 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_2/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:23.853 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_2/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:44:23.853 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_2/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:23.853 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_2/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:44:23.853 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.GETATTR_DIRS1 (0 ms) 2023-09-22T13:44:23.853 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READ_DIRS1 2023-09-22T13:44:23.853 INFO:tasks.workunit.client.0.smithi069.stdout:read back objects in /bdirs1/dir_0 2023-09-22T13:44:23.853 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_0/sfile_0 2023-09-22T13:44:23.854 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:44:23.854 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_0/sfile_1 2023-09-22T13:44:23.856 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:44:23.856 INFO:tasks.workunit.client.0.smithi069.stdout:read back objects in /bdirs1/dir_1 2023-09-22T13:44:23.856 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_1/sfile_0 2023-09-22T13:44:23.857 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:44:23.857 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_1/sfile_1 2023-09-22T13:44:23.859 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:44:23.859 INFO:tasks.workunit.client.0.smithi069.stdout:read back objects in /bdirs1/dir_2 2023-09-22T13:44:23.859 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_2/sfile_0 2023-09-22T13:44:23.860 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:44:23.861 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_2/sfile_1 2023-09-22T13:44:23.862 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:44:23.862 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READ_DIRS1 (11 ms) 2023-09-22T13:44:23.862 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READF_SETUP1 2023-09-22T13:44:24.039 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READF_SETUP1 (177 ms) 2023-09-22T13:44:24.039 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READF_DIRS1 2023-09-22T13:44:24.039 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READF_DIRS1 (0 ms) 2023-09-22T13:44:24.039 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.WRITEF_DIRS1 2023-09-22T13:44:24.040 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.WRITEF_DIRS1 (0 ms) 2023-09-22T13:44:24.040 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RELEASE_DIRS1 2023-09-22T13:44:24.040 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:44:24.040 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:44:24.040 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:44:24.040 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:44:24.040 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:44:24.040 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:44:24.040 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:44:24.040 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:44:24.041 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:44:24.041 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:44:24.041 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:44:24.041 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:44:24.041 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RELEASE_DIRS1 (0 ms) 2023-09-22T13:44:24.041 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.HIER1 2023-09-22T13:44:24.047 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/f2 (f2): FILE 2023-09-22T13:44:24.047 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/f1 (f1): FILE 2023-09-22T13:44:24.051 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/f1 (f1): FILE 2023-09-22T13:44:24.053 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/d1 (d1): DIR 2023-09-22T13:44:24.057 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:44:24.057 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:44:24.058 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/quux (quux): FILE 2023-09-22T13:44:24.058 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/momma (momma): FILE 2023-09-22T13:44:24.059 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:44:24.059 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:44:24.060 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/frobozz (frobozz): FILE 2023-09-22T13:44:24.062 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/bear (bear): DIR 2023-09-22T13:44:24.063 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz (baz): DIR 2023-09-22T13:44:24.063 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar (bar): DIR 2023-09-22T13:44:24.063 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo (foo): DIR 2023-09-22T13:44:24.063 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot (nfsroot): DIR 2023-09-22T13:44:24.063 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.HIER1 (23 ms) 2023-09-22T13:44:24.063 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:44:24.063 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (0 ms) 2023-09-22T13:44:24.063 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:44:24.063 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:44:24.063 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:44:24.063 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:44:24.064 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:44:24.064 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:44:24.064 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:44:24.064 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:44:24.064 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:44:24.064 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:44:24.064 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:44:24.064 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:24.061+0000 7f861c91f540 -1 shutting down 2023-09-22T13:44:24.064 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:24.061+0000 7f851ffff700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156122, si_uid : 1000, si_addr0x3e8000261da, si_status0 2023-09-22T13:44:24.107 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SHUTDOWN (45 ms) 2023-09-22T13:44:24.107 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 24 tests from LibRGW (759 ms total) 2023-09-22T13:44:24.107 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:24.107 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:24.108 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 24 tests from 1 test suite ran. (759 ms total) 2023-09-22T13:44:24.108 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 24 tests. 2023-09-22T13:44:24.109 INFO:tasks.workunit.client.0.smithi069.stdout:phase 1.2 2023-09-22T13:44:24.124 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 24 tests from 1 test suite. 2023-09-22T13:44:24.124 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:24.125 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 24 tests from LibRGW 2023-09-22T13:44:24.125 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:44:24.132 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:24.132 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:24.233 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.INIT (109 ms) 2023-09-22T13:44:24.234 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT_NOROOT 2023-09-22T13:44:24.236 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT_NOROOT (2 ms) 2023-09-22T13:44:24.236 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:44:24.236 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT (0 ms) 2023-09-22T13:44:24.236 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SETUP_HIER1 2023-09-22T13:44:24.236 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SETUP_HIER1 (0 ms) 2023-09-22T13:44:24.236 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SETUP_DIRS1 2023-09-22T13:44:24.236 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SETUP_DIRS1 (0 ms) 2023-09-22T13:44:24.236 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SETATTR 2023-09-22T13:44:24.236 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SETATTR (0 ms) 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_CREATE_DIRS1 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_CREATE_DIRS1 (0 ms) 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_SETUP_RENAME1 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_SETUP_RENAME1 (0 ms) 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_INTRABUCKET_RENAME1 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_INTRABUCKET_RENAME1 (0 ms) 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_CROSSBUCKET_RENAME1 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_CROSSBUCKET_RENAME1 (0 ms) 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.GETATTR_DIRS1 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.GETATTR_DIRS1 (0 ms) 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READ_DIRS1 2023-09-22T13:44:24.237 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READ_DIRS1 (0 ms) 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READF_SETUP1 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READF_SETUP1 (0 ms) 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READF_DIRS1 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READF_DIRS1 (0 ms) 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.WRITEF_DIRS1 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.WRITEF_DIRS1 (0 ms) 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RELEASE_DIRS1 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RELEASE_DIRS1 (0 ms) 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.HIER1 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.HIER1 (0 ms) 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:44:24.238 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (0 ms) 2023-09-22T13:44:24.239 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:44:24.239 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:44:24.239 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:44:24.239 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:44:24.239 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:44:24.239 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:44:24.239 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:44:24.239 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:44:24.239 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:44:24.239 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:44:24.239 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:44:24.240 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:24.233+0000 7f210bc3d540 -1 shutting down 2023-09-22T13:44:24.240 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:24.233+0000 7f20137fe700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156206, si_uid : 1000, si_addr0x3e80002622e, si_status0 2023-09-22T13:44:24.334 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SHUTDOWN (99 ms) 2023-09-22T13:44:24.334 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 24 tests from LibRGW (210 ms total) 2023-09-22T13:44:24.334 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:24.335 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:24.335 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 24 tests from 1 test suite ran. (210 ms total) 2023-09-22T13:44:24.335 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 24 tests. 2023-09-22T13:44:24.336 INFO:tasks.workunit.client.0.smithi069.stdout:phase 1.3 2023-09-22T13:44:24.352 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 24 tests from 1 test suite. 2023-09-22T13:44:24.352 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:24.352 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 24 tests from LibRGW 2023-09-22T13:44:24.352 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:44:24.361 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:24.361 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:24.472 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.INIT (120 ms) 2023-09-22T13:44:24.472 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT_NOROOT 2023-09-22T13:44:24.473 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT_NOROOT (1 ms) 2023-09-22T13:44:24.474 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:44:24.474 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT (0 ms) 2023-09-22T13:44:24.474 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SETUP_HIER1 2023-09-22T13:44:24.479 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SETUP_HIER1 (7 ms) 2023-09-22T13:44:24.480 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SETUP_DIRS1 2023-09-22T13:44:24.580 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SETUP_DIRS1 (100 ms) 2023-09-22T13:44:24.580 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SETATTR 2023-09-22T13:44:24.581 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SETATTR (0 ms) 2023-09-22T13:44:24.581 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_CREATE_DIRS1 2023-09-22T13:44:24.581 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_CREATE_DIRS1 (0 ms) 2023-09-22T13:44:24.581 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_SETUP_RENAME1 2023-09-22T13:44:24.581 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_SETUP_RENAME1 (0 ms) 2023-09-22T13:44:24.581 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_INTRABUCKET_RENAME1 2023-09-22T13:44:24.581 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_INTRABUCKET_RENAME1 (0 ms) 2023-09-22T13:44:24.581 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RGW_CROSSBUCKET_RENAME1 2023-09-22T13:44:24.581 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RGW_CROSSBUCKET_RENAME1 (0 ms) 2023-09-22T13:44:24.581 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.GETATTR_DIRS1 2023-09-22T13:44:24.581 INFO:tasks.workunit.client.0.smithi069.stdout:scanning objects in /bdirs1/dir_0 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_0/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_0/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_0/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_0/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout:scanning objects in /bdirs1/dir_1 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_1/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_1/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_1/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_1/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout:scanning objects in /bdirs1/dir_2 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_2/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:24.582 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_2/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:44:24.583 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_2/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:44:24.583 INFO:tasks.workunit.client.0.smithi069.stdout: /bdirs1/dir_2/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:44:24.583 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.GETATTR_DIRS1 (0 ms) 2023-09-22T13:44:24.583 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READ_DIRS1 2023-09-22T13:44:24.583 INFO:tasks.workunit.client.0.smithi069.stdout:read back objects in /bdirs1/dir_0 2023-09-22T13:44:24.583 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_0/sfile_0 2023-09-22T13:44:24.583 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:44:24.584 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_0/sfile_1 2023-09-22T13:44:24.584 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:44:24.584 INFO:tasks.workunit.client.0.smithi069.stdout:read back objects in /bdirs1/dir_1 2023-09-22T13:44:24.584 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_1/sfile_0 2023-09-22T13:44:24.585 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:44:24.585 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_1/sfile_1 2023-09-22T13:44:24.586 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:44:24.587 INFO:tasks.workunit.client.0.smithi069.stdout:read back objects in /bdirs1/dir_2 2023-09-22T13:44:24.587 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_2/sfile_0 2023-09-22T13:44:24.588 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:44:24.588 INFO:tasks.workunit.client.0.smithi069.stdout:reading 0,256 dir_2/sfile_1 2023-09-22T13:44:24.590 INFO:tasks.workunit.client.0.smithi069.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:44:24.590 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READ_DIRS1 (9 ms) 2023-09-22T13:44:24.590 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READF_SETUP1 2023-09-22T13:44:24.590 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READF_SETUP1 (0 ms) 2023-09-22T13:44:24.590 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READF_DIRS1 2023-09-22T13:44:24.590 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READF_DIRS1 (0 ms) 2023-09-22T13:44:24.590 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.WRITEF_DIRS1 2023-09-22T13:44:24.590 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.WRITEF_DIRS1 (0 ms) 2023-09-22T13:44:24.590 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.RELEASE_DIRS1 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:44:24.591 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.RELEASE_DIRS1 (0 ms) 2023-09-22T13:44:24.592 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.HIER1 2023-09-22T13:44:24.598 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/f2 (f2): FILE 2023-09-22T13:44:24.598 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/f1 (f1): FILE 2023-09-22T13:44:24.602 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/f1 (f1): FILE 2023-09-22T13:44:24.604 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/d1 (d1): DIR 2023-09-22T13:44:24.608 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:44:24.608 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:44:24.609 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/quux (quux): FILE 2023-09-22T13:44:24.609 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/momma (momma): FILE 2023-09-22T13:44:24.610 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:44:24.610 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:44:24.611 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/frobozz (frobozz): FILE 2023-09-22T13:44:24.614 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz/bear (bear): DIR 2023-09-22T13:44:24.614 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar/baz (baz): DIR 2023-09-22T13:44:24.614 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo/bar (bar): DIR 2023-09-22T13:44:24.614 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot/foo (foo): DIR 2023-09-22T13:44:24.614 INFO:tasks.workunit.client.0.smithi069.stdout:/nfsroot (nfsroot): DIR 2023-09-22T13:44:24.614 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.HIER1 (24 ms) 2023-09-22T13:44:24.614 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:44:24.614 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (0 ms) 2023-09-22T13:44:24.614 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:44:24.615 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:44:24.615 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:44:24.615 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:44:24.615 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:44:24.615 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:44:24.615 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:44:24.615 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:44:24.615 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:44:24.615 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:44:24.615 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:44:24.615 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:24.612+0000 7f7034063540 -1 shutting down 2023-09-22T13:44:24.616 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:24.612+0000 7f6f3b7fe700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156290, si_uid : 1000, si_addr0x3e800026282, si_status0 2023-09-22T13:44:24.673 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SHUTDOWN (60 ms) 2023-09-22T13:44:24.673 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 24 tests from LibRGW (321 ms total) 2023-09-22T13:44:24.673 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:24.673 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:24.674 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 24 tests from 1 test suite ran. (321 ms total) 2023-09-22T13:44:24.674 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 24 tests. 2023-09-22T13:44:24.675 INFO:tasks.workunit.client.0.smithi069.stdout:phase 2.1 2023-09-22T13:44:24.690 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 9 tests from 1 test suite. 2023-09-22T13:44:24.691 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:24.691 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 9 tests from LibRGW 2023-09-22T13:44:24.691 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:44:24.699 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:24.699 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:24.788 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.INIT (98 ms) 2023-09-22T13:44:24.788 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:44:24.789 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:44:24.789 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:44:24.801 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CREATE_BUCKET (12 ms) 2023-09-22T13:44:24.801 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:44:24.801 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_BUCKET (0 ms) 2023-09-22T13:44:24.801 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CREATE_BUCKET_MULTI 2023-09-22T13:44:24.816 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave0 2023-09-22T13:44:24.830 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave1 2023-09-22T13:44:24.842 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave2 2023-09-22T13:44:24.853 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave3 2023-09-22T13:44:24.863 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave4 2023-09-22T13:44:24.876 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave5 2023-09-22T13:44:24.886 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave6 2023-09-22T13:44:24.895 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave7 2023-09-22T13:44:24.906 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave8 2023-09-22T13:44:24.919 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave9 2023-09-22T13:44:24.919 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CREATE_BUCKET_MULTI (118 ms) 2023-09-22T13:44:24.919 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_BUCKET_MULTI 2023-09-22T13:44:25.271 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_BUCKET_MULTI (352 ms) 2023-09-22T13:44:25.272 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:44:25.272 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:44:25.272 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:44:25.272 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:44:25.272 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:44:25.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:25.270+0000 7f0523d2e540 -1 shutting down 2023-09-22T13:44:25.272 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:25.270+0000 7f042b7fe700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156374, si_uid : 1000, si_addr0x3e8000262d6, si_status0 2023-09-22T13:44:25.389 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SHUTDOWN (118 ms) 2023-09-22T13:44:25.389 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 9 tests from LibRGW (699 ms total) 2023-09-22T13:44:25.389 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:25.389 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:25.389 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 9 tests from 1 test suite ran. (699 ms total) 2023-09-22T13:44:25.389 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 9 tests. 2023-09-22T13:44:25.391 INFO:tasks.workunit.client.0.smithi069.stdout:phase 2.2 2023-09-22T13:44:25.406 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 9 tests from 1 test suite. 2023-09-22T13:44:25.406 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:25.406 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 9 tests from LibRGW 2023-09-22T13:44:25.406 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:44:25.415 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:25.415 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:25.522 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.INIT (116 ms) 2023-09-22T13:44:25.522 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:44:25.523 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:44:25.523 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:44:25.523 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CREATE_BUCKET (0 ms) 2023-09-22T13:44:25.523 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:44:25.559 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_BUCKET (36 ms) 2023-09-22T13:44:25.559 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CREATE_BUCKET_MULTI 2023-09-22T13:44:25.567 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave0 2023-09-22T13:44:25.576 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave1 2023-09-22T13:44:25.585 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave2 2023-09-22T13:44:25.593 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave3 2023-09-22T13:44:25.602 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave4 2023-09-22T13:44:25.610 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave5 2023-09-22T13:44:25.619 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave6 2023-09-22T13:44:25.629 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave7 2023-09-22T13:44:25.642 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave8 2023-09-22T13:44:25.654 INFO:tasks.workunit.client.0.smithi069.stdout:created: sorrydave9 2023-09-22T13:44:25.654 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CREATE_BUCKET_MULTI (95 ms) 2023-09-22T13:44:25.654 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_BUCKET_MULTI 2023-09-22T13:44:26.015 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_BUCKET_MULTI (361 ms) 2023-09-22T13:44:26.015 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:44:26.016 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:44:26.016 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:44:26.016 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:44:26.016 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:44:26.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:26.013+0000 7fda49de8540 -1 shutting down 2023-09-22T13:44:26.016 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:26.013+0000 7fd95dffb700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156458, si_uid : 1000, si_addr0x3e80002632a, si_status0 2023-09-22T13:44:26.123 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SHUTDOWN (108 ms) 2023-09-22T13:44:26.123 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 9 tests from LibRGW (717 ms total) 2023-09-22T13:44:26.123 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:26.123 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:26.124 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 9 tests from 1 test suite ran. (717 ms total) 2023-09-22T13:44:26.124 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 9 tests. 2023-09-22T13:44:26.125 INFO:tasks.workunit.client.0.smithi069.stdout:phase 3.1 2023-09-22T13:44:26.874 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 13 tests from 1 test suite. 2023-09-22T13:44:26.874 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:26.874 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 13 tests from LibRGW 2023-09-22T13:44:26.874 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:44:26.882 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:26.883 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:27.190 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.INIT (316 ms) 2023-09-22T13:44:27.190 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:44:27.191 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT (2 ms) 2023-09-22T13:44:27.192 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:44:27.203 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CREATE_BUCKET (11 ms) 2023-09-22T13:44:27.203 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:44:27.207 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (4 ms) 2023-09-22T13:44:27.207 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:44:27.209 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (2 ms) 2023-09-22T13:44:27.209 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LARGE1 2023-09-22T13:44:27.666 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LARGE1 (458 ms) 2023-09-22T13:44:27.667 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LARGE2 2023-09-22T13:44:27.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LARGE2 (75 ms) 2023-09-22T13:44:27.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:44:27.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:44:27.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:44:27.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_OBJECT (0 ms) 2023-09-22T13:44:27.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:44:27.742 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_BUCKET (0 ms) 2023-09-22T13:44:27.743 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:44:27.743 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:44:27.743 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:44:27.743 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:44:27.743 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:44:27.743 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:27.740+0000 7fc95b83f540 -1 shutting down 2023-09-22T13:44:27.744 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:27.740+0000 7fc86d7fa700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156542, si_uid : 1000, si_addr0x3e80002637e, si_status0 2023-09-22T13:44:27.791 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SHUTDOWN (49 ms) 2023-09-22T13:44:27.791 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 13 tests from LibRGW (917 ms total) 2023-09-22T13:44:27.791 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:27.792 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:27.792 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 13 tests from 1 test suite ran. (917 ms total) 2023-09-22T13:44:27.792 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 13 tests. 2023-09-22T13:44:27.795 INFO:tasks.workunit.client.0.smithi069.stdout:phase 3.2 2023-09-22T13:44:28.542 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 13 tests from 1 test suite. 2023-09-22T13:44:28.543 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:28.543 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 13 tests from LibRGW 2023-09-22T13:44:28.543 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:44:28.551 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:28.551 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:28.662 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.INIT (120 ms) 2023-09-22T13:44:28.662 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:44:28.663 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:44:28.663 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:44:28.664 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CREATE_BUCKET (0 ms) 2023-09-22T13:44:28.664 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:44:28.669 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (6 ms) 2023-09-22T13:44:28.669 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:44:28.675 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (6 ms) 2023-09-22T13:44:28.675 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LARGE1 2023-09-22T13:44:28.763 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LARGE1 (88 ms) 2023-09-22T13:44:28.763 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LARGE2 2023-09-22T13:44:28.763 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LARGE2 (0 ms) 2023-09-22T13:44:28.763 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:44:28.763 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:44:28.763 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:44:28.768 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_OBJECT (5 ms) 2023-09-22T13:44:28.768 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:44:28.801 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_BUCKET (33 ms) 2023-09-22T13:44:28.801 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:44:28.801 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:44:28.801 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:44:28.801 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:44:28.802 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:44:28.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:28.799+0000 7f7ce339c540 -1 shutting down 2023-09-22T13:44:28.802 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:28.799+0000 7f7bf57fa700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156626, si_uid : 1000, si_addr0x3e8000263d2, si_status0 2023-09-22T13:44:28.863 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SHUTDOWN (62 ms) 2023-09-22T13:44:28.863 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 13 tests from LibRGW (321 ms total) 2023-09-22T13:44:28.863 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:28.863 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:28.863 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 13 tests from 1 test suite ran. (321 ms total) 2023-09-22T13:44:28.863 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 13 tests. 2023-09-22T13:44:28.866 INFO:tasks.workunit.client.0.smithi069.stdout:phase 4.1 2023-09-22T13:44:28.881 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 11 tests from 1 test suite. 2023-09-22T13:44:28.881 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:28.881 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 11 tests from LibRGW 2023-09-22T13:44:28.882 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.TVAR 2023-09-22T13:44:28.882 INFO:tasks.workunit.client.0.smithi069.stdout:read i1: 64001 2023-09-22T13:44:28.882 INFO:tasks.workunit.client.0.smithi069.stdout:read s1: blunderbuss 2023-09-22T13:44:28.882 INFO:tasks.workunit.client.0.smithi069.stdout:read s3: [ OK ] LibRGW.TVAR (0 ms) 2023-09-22T13:44:28.882 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:44:28.892 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:28.892 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:28.990 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.INIT (109 ms) 2023-09-22T13:44:28.990 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:44:28.992 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT (2 ms) 2023-09-22T13:44:28.992 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:44:29.017 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (25 ms) 2023-09-22T13:44:29.017 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:44:29.668 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_0 2023-09-22T13:44:29.668 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_1 2023-09-22T13:44:29.668 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_2 2023-09-22T13:44:29.668 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_3 2023-09-22T13:44:29.668 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_4 2023-09-22T13:44:29.668 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_5 2023-09-22T13:44:29.669 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_6 2023-09-22T13:44:29.669 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_7 2023-09-22T13:44:29.669 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_8 2023-09-22T13:44:29.669 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_9 2023-09-22T13:44:29.669 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_10 2023-09-22T13:44:29.669 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_11 2023-09-22T13:44:29.669 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_12 2023-09-22T13:44:29.669 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_13 2023-09-22T13:44:29.669 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_14 2023-09-22T13:44:29.669 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_15 2023-09-22T13:44:29.669 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_16 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_17 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_18 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_19 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_20 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_21 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_22 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_23 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_24 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_25 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_26 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_27 2023-09-22T13:44:29.670 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_28 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_29 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_30 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_31 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_32 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_33 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_34 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_35 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_36 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_37 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_38 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_39 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_40 2023-09-22T13:44:29.671 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_41 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_42 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_43 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_44 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_45 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_46 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_47 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_48 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_49 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_50 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_51 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_52 2023-09-22T13:44:29.672 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_53 2023-09-22T13:44:29.673 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_54 2023-09-22T13:44:29.673 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_55 2023-09-22T13:44:29.673 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_56 2023-09-22T13:44:29.673 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_57 2023-09-22T13:44:29.673 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_58 2023-09-22T13:44:29.673 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_59 2023-09-22T13:44:29.673 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_60 2023-09-22T13:44:29.673 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_61 2023-09-22T13:44:29.673 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_62 2023-09-22T13:44:29.673 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_63 2023-09-22T13:44:29.673 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_64 2023-09-22T13:44:29.674 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_65 2023-09-22T13:44:29.674 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_66 2023-09-22T13:44:29.674 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_67 2023-09-22T13:44:29.674 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_68 2023-09-22T13:44:29.674 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_69 2023-09-22T13:44:29.674 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_70 2023-09-22T13:44:29.674 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_71 2023-09-22T13:44:29.674 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_72 2023-09-22T13:44:29.674 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_73 2023-09-22T13:44:29.674 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_74 2023-09-22T13:44:29.674 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_75 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_76 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_77 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_78 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_79 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_80 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_81 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_82 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_83 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_84 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_85 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_86 2023-09-22T13:44:29.675 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_87 2023-09-22T13:44:29.676 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_88 2023-09-22T13:44:29.676 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_89 2023-09-22T13:44:29.676 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_90 2023-09-22T13:44:29.676 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_91 2023-09-22T13:44:29.676 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_92 2023-09-22T13:44:29.676 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_93 2023-09-22T13:44:29.676 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_94 2023-09-22T13:44:29.676 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_95 2023-09-22T13:44:29.676 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_96 2023-09-22T13:44:29.676 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_97 2023-09-22T13:44:29.677 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_98 2023-09-22T13:44:29.677 INFO:tasks.workunit.client.0.smithi069.stdout:created: dmarker:f_99 2023-09-22T13:44:29.677 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (651 ms) 2023-09-22T13:44:29.677 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:44:29.678 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=. flags=4 2023-09-22T13:44:29.678 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=.. flags=4 2023-09-22T13:44:29.678 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_0 flags=8 2023-09-22T13:44:29.678 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_1 flags=8 2023-09-22T13:44:29.678 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_10 flags=8 2023-09-22T13:44:29.679 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_11 flags=8 2023-09-22T13:44:29.679 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_12 flags=8 2023-09-22T13:44:29.679 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_13 flags=8 2023-09-22T13:44:29.679 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_14 flags=8 2023-09-22T13:44:29.679 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_15 flags=8 2023-09-22T13:44:29.679 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_16 flags=8 2023-09-22T13:44:29.679 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_17 flags=8 2023-09-22T13:44:29.679 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_18 flags=8 2023-09-22T13:44:29.679 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_19 flags=8 2023-09-22T13:44:29.679 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_2 flags=8 2023-09-22T13:44:29.680 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_20 flags=8 2023-09-22T13:44:29.680 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_21 flags=8 2023-09-22T13:44:29.680 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_22 flags=8 2023-09-22T13:44:29.680 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_23 flags=8 2023-09-22T13:44:29.680 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_24 flags=8 2023-09-22T13:44:29.680 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_25 flags=8 2023-09-22T13:44:29.680 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_26 flags=8 2023-09-22T13:44:29.680 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_27 flags=8 2023-09-22T13:44:29.680 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_28 flags=8 2023-09-22T13:44:29.680 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_29 flags=8 2023-09-22T13:44:29.681 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_3 flags=8 2023-09-22T13:44:29.681 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_30 flags=8 2023-09-22T13:44:29.681 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_31 flags=8 2023-09-22T13:44:29.681 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_32 flags=8 2023-09-22T13:44:29.681 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_33 flags=8 2023-09-22T13:44:29.681 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_34 flags=8 2023-09-22T13:44:29.681 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_35 flags=8 2023-09-22T13:44:29.681 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_36 flags=8 2023-09-22T13:44:29.682 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_37 flags=8 2023-09-22T13:44:29.682 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_38 flags=8 2023-09-22T13:44:29.682 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_39 flags=8 2023-09-22T13:44:29.682 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_4 flags=8 2023-09-22T13:44:29.682 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_40 flags=8 2023-09-22T13:44:29.682 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_41 flags=8 2023-09-22T13:44:29.682 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_42 flags=8 2023-09-22T13:44:29.682 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_43 flags=8 2023-09-22T13:44:29.682 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_44 flags=8 2023-09-22T13:44:29.682 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_45 flags=8 2023-09-22T13:44:29.682 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_46 flags=8 2023-09-22T13:44:29.683 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_47 flags=8 2023-09-22T13:44:29.683 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_48 flags=8 2023-09-22T13:44:29.683 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_49 flags=8 2023-09-22T13:44:29.683 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_5 flags=8 2023-09-22T13:44:29.683 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_50 flags=8 2023-09-22T13:44:29.683 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_51 flags=8 2023-09-22T13:44:29.683 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_52 flags=8 2023-09-22T13:44:29.683 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_53 flags=8 2023-09-22T13:44:29.683 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_54 flags=8 2023-09-22T13:44:29.684 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_55 flags=8 2023-09-22T13:44:29.684 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_56 flags=8 2023-09-22T13:44:29.684 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_57 flags=8 2023-09-22T13:44:29.684 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_58 flags=8 2023-09-22T13:44:29.684 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_59 flags=8 2023-09-22T13:44:29.684 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_6 flags=8 2023-09-22T13:44:29.684 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_60 flags=8 2023-09-22T13:44:29.684 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_61 flags=8 2023-09-22T13:44:29.684 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_62 flags=8 2023-09-22T13:44:29.685 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_63 flags=8 2023-09-22T13:44:29.685 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_64 flags=8 2023-09-22T13:44:29.685 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_65 flags=8 2023-09-22T13:44:29.685 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_66 flags=8 2023-09-22T13:44:29.685 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_67 flags=8 2023-09-22T13:44:29.685 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_68 flags=8 2023-09-22T13:44:29.685 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_69 flags=8 2023-09-22T13:44:29.685 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_7 flags=8 2023-09-22T13:44:29.685 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_70 flags=8 2023-09-22T13:44:29.685 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_71 flags=8 2023-09-22T13:44:29.685 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_72 flags=8 2023-09-22T13:44:29.686 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_73 flags=8 2023-09-22T13:44:29.686 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_74 flags=8 2023-09-22T13:44:29.686 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_75 flags=8 2023-09-22T13:44:29.686 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_76 flags=8 2023-09-22T13:44:29.686 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_77 flags=8 2023-09-22T13:44:29.686 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_78 flags=8 2023-09-22T13:44:29.686 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_79 flags=8 2023-09-22T13:44:29.686 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_8 flags=8 2023-09-22T13:44:29.687 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_80 flags=8 2023-09-22T13:44:29.687 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_81 flags=8 2023-09-22T13:44:29.687 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_82 flags=8 2023-09-22T13:44:29.687 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_83 flags=8 2023-09-22T13:44:29.687 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_84 flags=8 2023-09-22T13:44:29.687 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_85 flags=8 2023-09-22T13:44:29.687 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_86 flags=8 2023-09-22T13:44:29.687 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_87 flags=8 2023-09-22T13:44:29.687 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_88 flags=8 2023-09-22T13:44:29.688 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_89 flags=8 2023-09-22T13:44:29.688 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_9 flags=8 2023-09-22T13:44:29.688 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_90 flags=8 2023-09-22T13:44:29.688 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_91 flags=8 2023-09-22T13:44:29.688 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_92 flags=8 2023-09-22T13:44:29.688 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_93 flags=8 2023-09-22T13:44:29.688 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_94 flags=8 2023-09-22T13:44:29.688 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_95 flags=8 2023-09-22T13:44:29.688 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_96 flags=8 2023-09-22T13:44:29.688 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_97 flags=8 2023-09-22T13:44:29.688 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_98 flags=8 2023-09-22T13:44:29.689 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_99 flags=8 2023-09-22T13:44:29.689 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_READDIR (2 ms) 2023-09-22T13:44:29.689 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER2_READDIR 2023-09-22T13:44:29.689 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_0 flags=8 2023-09-22T13:44:29.689 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_1 flags=8 2023-09-22T13:44:29.689 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_10 flags=8 2023-09-22T13:44:29.689 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_11 flags=8 2023-09-22T13:44:29.689 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_12 flags=8 2023-09-22T13:44:29.689 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_13 flags=8 2023-09-22T13:44:29.689 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_14 flags=8 2023-09-22T13:44:29.690 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_15 flags=8 2023-09-22T13:44:29.690 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_16 flags=8 2023-09-22T13:44:29.690 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_17 flags=8 2023-09-22T13:44:29.690 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_18 flags=8 2023-09-22T13:44:29.690 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_19 flags=8 2023-09-22T13:44:29.690 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_2 flags=8 2023-09-22T13:44:29.690 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_20 flags=8 2023-09-22T13:44:29.690 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_21 flags=8 2023-09-22T13:44:29.690 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_22 flags=8 2023-09-22T13:44:29.690 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_23 flags=8 2023-09-22T13:44:29.691 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_24 flags=8 2023-09-22T13:44:29.691 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_25 flags=8 2023-09-22T13:44:29.691 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_26 flags=8 2023-09-22T13:44:29.691 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_27 flags=8 2023-09-22T13:44:29.691 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_28 flags=8 2023-09-22T13:44:29.691 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_29 flags=8 2023-09-22T13:44:29.691 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_3 flags=8 2023-09-22T13:44:29.691 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_30 flags=8 2023-09-22T13:44:29.691 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_31 flags=8 2023-09-22T13:44:29.691 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_32 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_33 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_34 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_35 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_36 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_37 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_38 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_39 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_4 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_40 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_41 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_42 flags=8 2023-09-22T13:44:29.692 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_43 flags=8 2023-09-22T13:44:29.693 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_44 flags=8 2023-09-22T13:44:29.693 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_45 flags=8 2023-09-22T13:44:29.693 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_46 flags=8 2023-09-22T13:44:29.693 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_47 flags=8 2023-09-22T13:44:29.693 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_48 flags=8 2023-09-22T13:44:29.693 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_49 flags=8 2023-09-22T13:44:29.693 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_5 flags=8 2023-09-22T13:44:29.693 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_50 flags=8 2023-09-22T13:44:29.693 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_51 flags=8 2023-09-22T13:44:29.693 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_52 flags=8 2023-09-22T13:44:29.693 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_53 flags=8 2023-09-22T13:44:29.694 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_54 flags=8 2023-09-22T13:44:29.694 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_55 flags=8 2023-09-22T13:44:29.694 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_56 flags=8 2023-09-22T13:44:29.694 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_57 flags=8 2023-09-22T13:44:29.694 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_58 flags=8 2023-09-22T13:44:29.694 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_59 flags=8 2023-09-22T13:44:29.694 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_6 flags=8 2023-09-22T13:44:29.694 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_60 flags=8 2023-09-22T13:44:29.694 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_61 flags=8 2023-09-22T13:44:29.694 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_62 flags=8 2023-09-22T13:44:29.694 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_63 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_64 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_65 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_66 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_67 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_68 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_69 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_7 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_70 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_71 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_72 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_73 flags=8 2023-09-22T13:44:29.695 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_74 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_75 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_76 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_77 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_78 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_79 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_8 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_80 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_81 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_82 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_83 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_84 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_85 flags=8 2023-09-22T13:44:29.696 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_86 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_87 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_88 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_89 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_9 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_90 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_91 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_92 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_93 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_94 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_95 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_96 flags=8 2023-09-22T13:44:29.697 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_97 flags=8 2023-09-22T13:44:29.698 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_98 flags=8 2023-09-22T13:44:29.698 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_99 flags=8 2023-09-22T13:44:29.698 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER2_READDIR (2 ms) 2023-09-22T13:44:29.698 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:44:29.698 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:44:29.698 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:44:29.698 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:44:29.698 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:44:29.698 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:44:29.698 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:44:29.699 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:29.671+0000 7f7719b3f540 -1 shutting down 2023-09-22T13:44:29.699 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:29.671+0000 7f7631ffb700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156711, si_uid : 1000, si_addr0x3e800026427, si_status0 2023-09-22T13:44:29.699 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SHUTDOWN (20 ms) 2023-09-22T13:44:29.699 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 11 tests from LibRGW (811 ms total) 2023-09-22T13:44:29.699 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:29.699 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:29.699 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 11 tests from 1 test suite ran. (811 ms total) 2023-09-22T13:44:29.699 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 11 tests. 2023-09-22T13:44:29.700 INFO:tasks.workunit.client.0.smithi069.stdout:phase 4.2 2023-09-22T13:44:29.709 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 11 tests from 1 test suite. 2023-09-22T13:44:29.709 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:29.709 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 11 tests from LibRGW 2023-09-22T13:44:29.709 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.TVAR 2023-09-22T13:44:29.709 INFO:tasks.workunit.client.0.smithi069.stdout:read i1: 64001 2023-09-22T13:44:29.710 INFO:tasks.workunit.client.0.smithi069.stdout:read s1: blunderbuss 2023-09-22T13:44:29.710 INFO:tasks.workunit.client.0.smithi069.stdout:read s3: [ OK ] LibRGW.TVAR (0 ms) 2023-09-22T13:44:29.710 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:44:29.717 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:29.717 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:29.812 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.INIT (103 ms) 2023-09-22T13:44:29.812 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:44:29.814 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT (2 ms) 2023-09-22T13:44:29.814 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:44:29.826 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (12 ms) 2023-09-22T13:44:29.826 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:44:29.826 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:44:29.827 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:44:29.827 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:44:29.827 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER2_READDIR 2023-09-22T13:44:29.829 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_0 flags=8 2023-09-22T13:44:29.829 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_1 flags=8 2023-09-22T13:44:29.829 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_10 flags=8 2023-09-22T13:44:29.829 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_11 flags=8 2023-09-22T13:44:29.829 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_12 flags=8 2023-09-22T13:44:29.829 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_13 flags=8 2023-09-22T13:44:29.829 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_14 flags=8 2023-09-22T13:44:29.829 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_15 flags=8 2023-09-22T13:44:29.829 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_16 flags=8 2023-09-22T13:44:29.829 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_17 flags=8 2023-09-22T13:44:29.829 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_18 flags=8 2023-09-22T13:44:29.830 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_19 flags=8 2023-09-22T13:44:29.830 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_2 flags=8 2023-09-22T13:44:29.830 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_20 flags=8 2023-09-22T13:44:29.830 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_21 flags=8 2023-09-22T13:44:29.830 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_22 flags=8 2023-09-22T13:44:29.830 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_23 flags=8 2023-09-22T13:44:29.830 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_24 flags=8 2023-09-22T13:44:29.830 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_25 flags=8 2023-09-22T13:44:29.830 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_26 flags=8 2023-09-22T13:44:29.830 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_27 flags=8 2023-09-22T13:44:29.830 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_28 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_29 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_3 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_30 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_31 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_32 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_33 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_34 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_35 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_36 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_37 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_38 flags=8 2023-09-22T13:44:29.831 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_39 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_4 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_40 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_41 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_42 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_43 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_44 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_45 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_46 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_47 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_48 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_49 flags=8 2023-09-22T13:44:29.832 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_5 flags=8 2023-09-22T13:44:29.833 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_50 flags=8 2023-09-22T13:44:29.833 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_51 flags=8 2023-09-22T13:44:29.833 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_52 flags=8 2023-09-22T13:44:29.833 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_53 flags=8 2023-09-22T13:44:29.833 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_54 flags=8 2023-09-22T13:44:29.833 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_55 flags=8 2023-09-22T13:44:29.833 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_56 flags=8 2023-09-22T13:44:29.833 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_57 flags=8 2023-09-22T13:44:29.833 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_58 flags=8 2023-09-22T13:44:29.833 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_59 flags=8 2023-09-22T13:44:29.834 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_6 flags=8 2023-09-22T13:44:29.834 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_60 flags=8 2023-09-22T13:44:29.834 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_61 flags=8 2023-09-22T13:44:29.834 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_62 flags=8 2023-09-22T13:44:29.834 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_63 flags=8 2023-09-22T13:44:29.834 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_64 flags=8 2023-09-22T13:44:29.834 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_65 flags=8 2023-09-22T13:44:29.834 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_66 flags=8 2023-09-22T13:44:29.834 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_67 flags=8 2023-09-22T13:44:29.834 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_68 flags=8 2023-09-22T13:44:29.834 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_69 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_7 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_70 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_71 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_72 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_73 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_74 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_75 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_76 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_77 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_78 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_79 flags=8 2023-09-22T13:44:29.835 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_8 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_80 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_81 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_82 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_83 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_84 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_85 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_86 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_87 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_88 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_89 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_9 flags=8 2023-09-22T13:44:29.836 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_90 flags=8 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_91 flags=8 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_92 flags=8 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_93 flags=8 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_94 flags=8 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_95 flags=8 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_96 flags=8 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_97 flags=8 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_98 flags=8 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_99 flags=8 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER2_READDIR (2 ms) 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:44:29.837 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:44:29.838 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:44:29.838 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:44:29.838 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:44:29.838 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:44:29.838 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:44:29.838 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:29.827+0000 7f693cda5540 -1 shutting down 2023-09-22T13:44:29.838 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:29.827+0000 7f6850ff9700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156795, si_uid : 1000, si_addr0x3e80002647b, si_status0 2023-09-22T13:44:29.913 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SHUTDOWN (85 ms) 2023-09-22T13:44:29.913 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 11 tests from LibRGW (204 ms total) 2023-09-22T13:44:29.913 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:29.913 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:29.913 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 11 tests from 1 test suite ran. (204 ms total) 2023-09-22T13:44:29.913 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 11 tests. 2023-09-22T13:44:29.915 INFO:tasks.workunit.client.0.smithi069.stdout:phase 5.1 2023-09-22T13:44:29.953 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 19 tests from 1 test suite. 2023-09-22T13:44:29.954 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:29.954 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 19 tests from LibRGW 2023-09-22T13:44:29.954 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:44:29.963 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:29.963 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:30.259 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.INIT (306 ms) 2023-09-22T13:44:30.259 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:44:30.260 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT (2 ms) 2023-09-22T13:44:30.260 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:44:30.277 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CREATE_BUCKET (16 ms) 2023-09-22T13:44:30.277 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:44:30.281 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (4 ms) 2023-09-22T13:44:30.281 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LIST_OBJECTS 2023-09-22T13:44:30.281 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LIST_OBJECTS (0 ms) 2023-09-22T13:44:30.281 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:44:30.283 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (2 ms) 2023-09-22T13:44:30.283 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.OBJ_OPEN 2023-09-22T13:44:30.283 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.OBJ_OPEN (0 ms) 2023-09-22T13:44:30.284 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.PUT_OBJECT 2023-09-22T13:44:30.287 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.PUT_OBJECT (4 ms) 2023-09-22T13:44:30.287 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.GET_OBJECT 2023-09-22T13:44:30.290 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.GET_OBJECT (3 ms) 2023-09-22T13:44:30.290 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:44:30.290 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:44:30.290 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.WRITE_READ_VERIFY 2023-09-22T13:44:30.290 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.WRITE_READ_VERIFY (0 ms) 2023-09-22T13:44:30.290 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.WRITEV 2023-09-22T13:44:30.290 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.WRITEV (0 ms) 2023-09-22T13:44:30.290 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READV 2023-09-22T13:44:30.290 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READV (0 ms) 2023-09-22T13:44:30.291 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READV_AFTER_WRITEV 2023-09-22T13:44:30.291 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READV_AFTER_WRITEV (0 ms) 2023-09-22T13:44:30.291 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:44:30.291 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_OBJECT (0 ms) 2023-09-22T13:44:30.291 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:44:30.291 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_BUCKET (0 ms) 2023-09-22T13:44:30.291 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:44:30.291 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:44:30.291 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:44:30.291 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:44:30.291 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:44:30.292 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:30.288+0000 7fc4c4176540 -1 shutting down 2023-09-22T13:44:30.292 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:30.288+0000 7fc3cb7fe700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156879, si_uid : 1000, si_addr0x3e8000264cf, si_status0 2023-09-22T13:44:30.360 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SHUTDOWN (70 ms) 2023-09-22T13:44:30.360 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 19 tests from LibRGW (407 ms total) 2023-09-22T13:44:30.360 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:30.360 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:30.360 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 19 tests from 1 test suite ran. (407 ms total) 2023-09-22T13:44:30.361 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 19 tests. 2023-09-22T13:44:30.362 INFO:tasks.workunit.client.0.smithi069.stdout:phase 5.2 2023-09-22T13:44:30.401 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] Running 19 tests from 1 test suite. 2023-09-22T13:44:30.401 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment set-up. 2023-09-22T13:44:30.401 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 19 tests from LibRGW 2023-09-22T13:44:30.401 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:44:30.407 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:30.407 INFO:tasks.workunit.client.0.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:30.506 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.INIT (105 ms) 2023-09-22T13:44:30.506 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:44:30.507 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.MOUNT (2 ms) 2023-09-22T13:44:30.508 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:44:30.508 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CREATE_BUCKET (0 ms) 2023-09-22T13:44:30.508 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:44:30.514 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (6 ms) 2023-09-22T13:44:30.514 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LIST_OBJECTS 2023-09-22T13:44:30.514 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LIST_OBJECTS (0 ms) 2023-09-22T13:44:30.514 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:44:30.515 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (0 ms) 2023-09-22T13:44:30.515 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.OBJ_OPEN 2023-09-22T13:44:30.515 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.OBJ_OPEN (0 ms) 2023-09-22T13:44:30.515 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.PUT_OBJECT 2023-09-22T13:44:30.515 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.PUT_OBJECT (0 ms) 2023-09-22T13:44:30.515 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.GET_OBJECT 2023-09-22T13:44:30.515 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.GET_OBJECT (0 ms) 2023-09-22T13:44:30.515 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:44:30.515 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:44:30.515 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.WRITE_READ_VERIFY 2023-09-22T13:44:30.515 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.WRITE_READ_VERIFY (0 ms) 2023-09-22T13:44:30.516 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.WRITEV 2023-09-22T13:44:30.516 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.WRITEV (0 ms) 2023-09-22T13:44:30.516 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READV 2023-09-22T13:44:30.516 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READV (0 ms) 2023-09-22T13:44:30.516 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.READV_AFTER_WRITEV 2023-09-22T13:44:30.516 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.READV_AFTER_WRITEV (0 ms) 2023-09-22T13:44:30.516 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:44:30.518 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_OBJECT (4 ms) 2023-09-22T13:44:30.518 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:44:30.556 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.DELETE_BUCKET (38 ms) 2023-09-22T13:44:30.556 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:44:30.556 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:44:30.556 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:44:30.556 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:44:30.556 INFO:tasks.workunit.client.0.smithi069.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:44:30.557 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:30.554+0000 7f3553c3a540 -1 shutting down 2023-09-22T13:44:30.557 INFO:tasks.workunit.client.0.smithi069.stderr:2023-09-22T13:44:30.554+0000 7f345b7fe700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156963, si_uid : 1000, si_addr0x3e800026523, si_status0 2023-09-22T13:44:30.607 INFO:tasks.workunit.client.0.smithi069.stdout:[ OK ] LibRGW.SHUTDOWN (51 ms) 2023-09-22T13:44:30.607 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] 19 tests from LibRGW (206 ms total) 2023-09-22T13:44:30.607 INFO:tasks.workunit.client.0.smithi069.stdout: 2023-09-22T13:44:30.607 INFO:tasks.workunit.client.0.smithi069.stdout:[----------] Global test environment tear-down 2023-09-22T13:44:30.607 INFO:tasks.workunit.client.0.smithi069.stdout:[==========] 19 tests from 1 test suite ran. (206 ms total) 2023-09-22T13:44:30.607 INFO:tasks.workunit.client.0.smithi069.stdout:[ PASSED ] 19 tests. 2023-09-22T13:44:30.610 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:44:30.610 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:44:30.638 INFO:tasks.workunit:Stopping ['cls/test_cls_lock.sh', 'cls/test_cls_log.sh', 'cls/test_cls_refcount.sh', 'cls/test_cls_rgw.sh', 'cls/test_cls_cmpomap.sh', 'cls/test_cls_2pc_queue.sh', 'rgw/test_rgw_gc_log.sh', 'rgw/test_rgw_obj.sh', 'rgw/test_rgw_throttle.sh', 'rgw/test_librgw_file.sh'] on client.0... 2023-09-22T13:44:30.638 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-22T13:44:30.932 DEBUG:teuthology.parallel:result is None 2023-09-22T13:44:30.933 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:44:30.960 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:44:30.960 DEBUG:teuthology.orchestra.run.smithi069:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:44:31.018 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:44:31.019 INFO:teuthology.run_tasks:Running task ragweed... 2023-09-22T13:44:31.032 DEBUG:tasks.ragweed:ragweed config is {'client.0': {'default-branch': 'ceph-pacific', 'rgw_server': 'client.0', 'stages': 'prepare,check'}} 2023-09-22T13:44:31.033 INFO:tasks.ragweed:Downloading ragweed... 2023-09-22T13:44:31.033 INFO:tasks.ragweed:Using branch 'wip-cfsnyder-bucket-check-pacific' for ragweed 2023-09-22T13:44:31.033 DEBUG:teuthology.orchestra.run.smithi069:> git clone -b wip-cfsnyder-bucket-check-pacific https://git.ceph.com/ragweed.git /home/ubuntu/cephtest/ragweed 2023-09-22T13:44:31.076 INFO:teuthology.orchestra.run.smithi069.stderr:Cloning into '/home/ubuntu/cephtest/ragweed'... 2023-09-22T13:44:31.151 INFO:teuthology.orchestra.run.smithi069.stderr:fatal: Remote branch wip-cfsnyder-bucket-check-pacific not found in upstream origin 2023-09-22T13:44:31.152 DEBUG:teuthology.orchestra.run:got remote process result: 128 2023-09-22T13:44:31.153 INFO:tasks.ragweed:Using branch 'ceph-pacific' for ragweed 2023-09-22T13:44:31.153 DEBUG:teuthology.orchestra.run.smithi069:> git clone -b ceph-pacific https://git.ceph.com/ragweed.git /home/ubuntu/cephtest/ragweed 2023-09-22T13:44:31.211 INFO:teuthology.orchestra.run.smithi069.stderr:Cloning into '/home/ubuntu/cephtest/ragweed'... 2023-09-22T13:44:31.364 INFO:tasks.ragweed:Creating rgw users... 2023-09-22T13:44:31.364 DEBUG:tasks.ragweed:Creating user ragweed.client.0 on client.0 2023-09-22T13:44:31.365 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user create --uid ragweed.client.0 --display-name 'Mr. ragweed.client.0' --access-key YTEQESUFQHRIOEDGJDEO --secret J7hD9ARXc8+ZaIQxmtRpSIqSctqZJUn/3k0gKDAgFNax78r2ArWH4w== --email ragweed.client.0+test@test.test --system false 2023-09-22T13:44:31.453 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:31.453 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:31.459 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.457+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.461 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.459+0000 7fb1bf586500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.461 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.459+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.462 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.460+0000 7fb1bf586500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.462 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.460+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.462 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.460+0000 7fb1bf586500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.462 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.460+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.463 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.461+0000 7fb1bf586500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:31.474 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.473+0000 7fb1bf586500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:44:31.474 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.473+0000 7fb1bf586500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:44:31.475 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.473+0000 7fb1bf586500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:44:31.475 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.473+0000 7fb1bf586500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:44:31.475 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.473+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.475 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.473+0000 7fb1bf586500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:31.475 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.473+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.476 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.473+0000 7fb1bf586500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:44:31.476 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.473+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.476 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.474+0000 7fb1bf586500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:31.476 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.474+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.477 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.474+0000 7fb1bf586500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:44:31.477 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.474+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.477 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.474+0000 7fb1bf586500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.478 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.474+0000 7fb1bf586500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:44:31.478 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.474+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.478 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.474+0000 7fb1bf586500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.478 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.474+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.478 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.475+0000 7fb1bf586500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:31.478 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.475+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.478 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.475+0000 7fb1bf586500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:44:31.478 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.475+0000 7fb1bf586500 20 zonegroup default 2023-09-22T13:44:31.478 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.475+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.478 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.476+0000 7fb1bf586500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.478 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.476+0000 7fb1bf586500 10 Cannot find current period zone using local zone 2023-09-22T13:44:31.479 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.476+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.479 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.476+0000 7fb1bf586500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.479 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.476+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.480 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.476+0000 7fb1bf586500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:31.480 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.476+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.480 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.477+0000 7fb1bf586500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:44:31.480 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.477+0000 7fb1bf586500 20 zone default found 2023-09-22T13:44:31.480 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.477+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.480 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.477+0000 7fb1bf586500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.480 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.477+0000 7fb1bf586500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.481 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.477+0000 7fb1bf586500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.481 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.477+0000 7fb1bf586500 20 started sync module instance, tier type = 2023-09-22T13:44:31.481 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.477+0000 7fb1bf586500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:44:31.488 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.486+0000 7fb1bf586500 20 add_watcher() i=0 2023-09-22T13:44:31.488 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.486+0000 7fb1bf586500 20 add_watcher() i=1 2023-09-22T13:44:31.488 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.486+0000 7fb1bf586500 20 add_watcher() i=2 2023-09-22T13:44:31.488 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.486+0000 7fb1bf586500 20 add_watcher() i=3 2023-09-22T13:44:31.488 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.486+0000 7fb1bf586500 20 add_watcher() i=4 2023-09-22T13:44:31.488 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.486+0000 7fb1bf586500 20 add_watcher() i=5 2023-09-22T13:44:31.488 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.486+0000 7fb1bf586500 20 add_watcher() i=6 2023-09-22T13:44:31.489 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.487+0000 7fb1bf586500 20 add_watcher() i=7 2023-09-22T13:44:31.489 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.487+0000 7fb1bf586500 2 all 8 watchers are set, enabling cache 2023-09-22T13:44:31.489 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.487+0000 7fb1ab269700 0 -- 172.21.15.69:0/2342324557 >> [v2:172.21.15.69:6816/145181,v1:172.21.15.69:6817/145181] conn(0x559a4b341830 msgr2=0x559a4b343c20 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:44:31.490 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.489+0000 7fb17bfe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:44:31.491 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.489+0000 7fb1bf586500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:44:31.491 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.489+0000 7fb1bf586500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:44:31.493 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.492+0000 7fb1aaa68700 0 -- 172.21.15.69:0/886628250 >> [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] conn(0x559a4b363b00 msgr2=0x559a4b36a180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:44:31.496 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.495+0000 7fb1bf586500 5 note: GC not initialized 2023-09-22T13:44:31.498 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.495+0000 7fb0eaffd700 20 reqs_thread_entry: start 2023-09-22T13:44:31.498 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.496+0000 7fb0e8ff9700 20 reqs_thread_entry: start 2023-09-22T13:44:31.498 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.496+0000 7fb1bf586500 20 init_complete bucket index max shards: 11 2023-09-22T13:44:31.498 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.496+0000 7fb1bf586500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:44:31.498 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.496+0000 7fb0d6ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:45:01 2023 2023-09-22T13:44:31.498 INFO:teuthology.orchestra.run.smithi069.stderr: 2023-09-22T13:44:31.498 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.496+0000 7fb0d67fc700 20 reqs_thread_entry: start 2023-09-22T13:44:31.498 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.496+0000 7fb1bf586500 20 get_system_obj_state: rctx=0x7ffd7669e878 obj=default.rgw.meta:users.uid:ragweed.client.0 state=0x559a4b204810 s->prefetch_data=0 2023-09-22T13:44:31.498 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.496+0000 7fb1bf586500 10 cache get: name=default.rgw.meta+users.uid+ragweed.client.0 : miss 2023-09-22T13:44:31.499 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.496+0000 7fb1bf586500 10 cache put: name=default.rgw.meta+users.uid+ragweed.client.0 info.flags=0x0 2023-09-22T13:44:31.499 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.496+0000 7fb1bf586500 10 adding default.rgw.meta+users.uid+ragweed.client.0 to cache LRU end 2023-09-22T13:44:31.499 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.496+0000 7fb1bf586500 20 get_system_obj_state: rctx=0x7ffd7669e888 obj=default.rgw.meta:users.email:ragweed.client.0+test@test.test state=0x559a4b204810 s->prefetch_data=0 2023-09-22T13:44:31.500 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.496+0000 7fb1bf586500 10 cache get: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test : miss 2023-09-22T13:44:31.500 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.497+0000 7fb1bf586500 10 cache put: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test info.flags=0x0 2023-09-22T13:44:31.500 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.497+0000 7fb1bf586500 10 adding default.rgw.meta+users.email+ragweed.client.0+test@test.test to cache LRU end 2023-09-22T13:44:31.500 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.497+0000 7fb1bf586500 20 get_system_obj_state: rctx=0x7ffd7669e888 obj=default.rgw.meta:users.keys:YTEQESUFQHRIOEDGJDEO state=0x559a4b204810 s->prefetch_data=0 2023-09-22T13:44:31.500 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.497+0000 7fb1bf586500 10 cache get: name=default.rgw.meta+users.keys+YTEQESUFQHRIOEDGJDEO : miss 2023-09-22T13:44:31.500 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.497+0000 7fb1bf586500 10 cache put: name=default.rgw.meta+users.keys+YTEQESUFQHRIOEDGJDEO info.flags=0x0 2023-09-22T13:44:31.500 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.497+0000 7fb1bf586500 10 adding default.rgw.meta+users.keys+YTEQESUFQHRIOEDGJDEO to cache LRU end 2023-09-22T13:44:31.500 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.497+0000 7fb1bf586500 20 get_system_obj_state: rctx=0x7ffd7669e188 obj=default.rgw.meta:users.keys:YTEQESUFQHRIOEDGJDEO state=0x559a4b204810 s->prefetch_data=0 2023-09-22T13:44:31.500 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.497+0000 7fb1bf586500 10 cache get: name=default.rgw.meta+users.keys+YTEQESUFQHRIOEDGJDEO : hit (negative entry) 2023-09-22T13:44:31.500 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.497+0000 7fb1bf586500 20 get_system_obj_state: rctx=0x7ffd7669e4c8 obj=default.rgw.meta:users.keys:YTEQESUFQHRIOEDGJDEO state=0x559a4b204810 s->prefetch_data=0 2023-09-22T13:44:31.500 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.497+0000 7fb1bf586500 10 cache get: name=default.rgw.meta+users.keys+YTEQESUFQHRIOEDGJDEO : hit (negative entry) 2023-09-22T13:44:31.501 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.498+0000 7fb1bf586500 10 cache put: name=default.rgw.meta+users.uid+ragweed.client.0 info.flags=0x17 2023-09-22T13:44:31.501 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.498+0000 7fb1bf586500 10 moving default.rgw.meta+users.uid+ragweed.client.0 to cache LRU end 2023-09-22T13:44:31.501 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.498+0000 7fb1bf586500 10 distributing notification oid=default.rgw.control:notify.4 cni=[op: 0, obj: default.rgw.meta:users.uid:ragweed.client.0, ofs0, ns] 2023-09-22T13:44:31.501 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.499+0000 7fb1a8a64700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1005022347304 cookie 94121175107408 notifier 5646 bl.length()=641 2023-09-22T13:44:31.501 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.499+0000 7fb1a8a64700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+ragweed.client.0 info.flags=0x17 2023-09-22T13:44:31.502 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.499+0000 7fb1a8a64700 10 rgw watcher librados: moving default.rgw.meta+users.uid+ragweed.client.0 to cache LRU end 2023-09-22T13:44:31.503 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.502+0000 7fb1bf586500 10 cache put: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test info.flags=0x7 2023-09-22T13:44:31.503 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.502+0000 7fb1bf586500 10 moving default.rgw.meta+users.email+ragweed.client.0+test@test.test to cache LRU end 2023-09-22T13:44:31.503 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.502+0000 7fb1bf586500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.email:ragweed.client.0+test@test.test, ofs0, ns] 2023-09-22T13:44:31.503 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.502+0000 7fb1a8a64700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1005022347270 cookie 94121175098384 notifier 5646 bl.length()=202 2023-09-22T13:44:31.503 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.502+0000 7fb1a8a64700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test info.flags=0x7 2023-09-22T13:44:31.503 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.502+0000 7fb1a8a64700 10 rgw watcher librados: moving default.rgw.meta+users.email+ragweed.client.0+test@test.test to cache LRU end 2023-09-22T13:44:31.505 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.504+0000 7fb1bf586500 10 cache put: name=default.rgw.meta+users.keys+YTEQESUFQHRIOEDGJDEO info.flags=0x7 2023-09-22T13:44:31.505 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.504+0000 7fb1bf586500 10 moving default.rgw.meta+users.keys+YTEQESUFQHRIOEDGJDEO to cache LRU end 2023-09-22T13:44:31.505 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.504+0000 7fb1bf586500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.keys:YTEQESUFQHRIOEDGJDEO, ofs0, ns] 2023-09-22T13:44:31.506 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.504+0000 7fb19bfff700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1005022347271 cookie 94121175098384 notifier 5646 bl.length()=190 2023-09-22T13:44:31.506 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.504+0000 7fb19bfff700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+YTEQESUFQHRIOEDGJDEO info.flags=0x7 2023-09-22T13:44:31.506 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.504+0000 7fb19bfff700 10 rgw watcher librados: moving default.rgw.meta+users.keys+YTEQESUFQHRIOEDGJDEO to cache LRU end 2023-09-22T13:44:31.506 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2023-09-22T13:44:31.506 INFO:teuthology.orchestra.run.smithi069.stdout: "user_id": "ragweed.client.0", 2023-09-22T13:44:31.506 INFO:teuthology.orchestra.run.smithi069.stdout: "display_name": "Mr. ragweed.client.0", 2023-09-22T13:44:31.506 INFO:teuthology.orchestra.run.smithi069.stdout: "email": "ragweed.client.0+test@test.test", 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: "suspended": 0, 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: "max_buckets": 1000, 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: "subusers": [], 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: "keys": [ 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: { 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: "user": "ragweed.client.0", 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: "access_key": "YTEQESUFQHRIOEDGJDEO", 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: "secret_key": "J7hD9ARXc8+ZaIQxmtRpSIqSctqZJUn/3k0gKDAgFNax78r2ArWH4w==" 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: "swift_keys": [], 2023-09-22T13:44:31.507 INFO:teuthology.orchestra.run.smithi069.stdout: "caps": [], 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: "op_mask": "read, write, delete", 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: "default_placement": "", 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: "default_storage_class": "", 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: "placement_tags": [], 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: "bucket_quota": { 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: "enabled": false, 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: "check_on_raw": false, 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: "max_size": -1, 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: "max_size_kb": 0, 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: "max_objects": -1 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:44:31.508 INFO:teuthology.orchestra.run.smithi069.stdout: "user_quota": { 2023-09-22T13:44:31.509 INFO:teuthology.orchestra.run.smithi069.stdout: "enabled": false, 2023-09-22T13:44:31.509 INFO:teuthology.orchestra.run.smithi069.stdout: "check_on_raw": false, 2023-09-22T13:44:31.509 INFO:teuthology.orchestra.run.smithi069.stdout: "max_size": -1, 2023-09-22T13:44:31.509 INFO:teuthology.orchestra.run.smithi069.stdout: "max_size_kb": 0, 2023-09-22T13:44:31.509 INFO:teuthology.orchestra.run.smithi069.stdout: "max_objects": -1 2023-09-22T13:44:31.509 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:44:31.509 INFO:teuthology.orchestra.run.smithi069.stdout: "temp_url_keys": [], 2023-09-22T13:44:31.509 INFO:teuthology.orchestra.run.smithi069.stdout: "type": "rgw", 2023-09-22T13:44:31.509 INFO:teuthology.orchestra.run.smithi069.stdout: "mfa_ids": [] 2023-09-22T13:44:31.509 INFO:teuthology.orchestra.run.smithi069.stdout:} 2023-09-22T13:44:31.510 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:44:31.510 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.506+0000 7fb1bf586500 20 remove_watcher() i=0 2023-09-22T13:44:31.510 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.506+0000 7fb1bf586500 2 removed watcher, disabling cache 2023-09-22T13:44:31.510 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.507+0000 7fb1bf586500 20 remove_watcher() i=1 2023-09-22T13:44:31.510 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.508+0000 7fb1bf586500 20 remove_watcher() i=2 2023-09-22T13:44:31.511 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.509+0000 7fb1bf586500 20 remove_watcher() i=3 2023-09-22T13:44:31.511 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.510+0000 7fb1bf586500 20 remove_watcher() i=4 2023-09-22T13:44:31.512 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.511+0000 7fb1bf586500 20 remove_watcher() i=5 2023-09-22T13:44:31.513 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.512+0000 7fb1bf586500 20 remove_watcher() i=6 2023-09-22T13:44:31.514 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.513+0000 7fb1bf586500 20 remove_watcher() i=7 2023-09-22T13:44:31.515 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.513+0000 7fb17bfe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:44:31.520 DEBUG:tasks.ragweed:Creating user sysuser.client.0 on client.0 2023-09-22T13:44:31.520 DEBUG:teuthology.orchestra.run.smithi069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user create --uid sysuser.client.0 --display-name 'Mr. sysuser.client.0' --access-key GRVTXEKBJSFREVTTDYYI --secret JhuFkyWULldI4j74AAvGNc1Isbx6dJmGFiSxH5/TyUyRZwrLuBp6VA== --email sysuser.client.0+test@test.test --system true 2023-09-22T13:44:31.568 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:44:31.568 INFO:teuthology.orchestra.run.smithi069.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:44:31.574 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.574+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.576 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.575+0000 7fdbe94c7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.576 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.575+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.577 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.576+0000 7fdbe94c7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.577 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.576+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.577 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.576+0000 7fdbe94c7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.577 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.576+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.578 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.577+0000 7fdbe94c7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:31.589 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.588+0000 7fdbe94c7500 20 RGWRados::pool_iterate: got zone_info.f54d190c-30d0-4b87-88c4-5bcc7de96de4 2023-09-22T13:44:31.589 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.588+0000 7fdbe94c7500 20 RGWRados::pool_iterate: got zonegroup_info.f2fb3710-6bf4-40b4-b2d8-9afc2276602f 2023-09-22T13:44:31.589 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.588+0000 7fdbe94c7500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:44:31.589 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.588+0000 7fdbe94c7500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:44:31.589 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.588+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.589 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.588+0000 7fdbe94c7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:31.589 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.588+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.590 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.589+0000 7fdbe94c7500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:44:31.590 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.589+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.591 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.589+0000 7fdbe94c7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:31.591 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.589+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.591 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.589+0000 7fdbe94c7500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:44:31.591 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.589+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.591 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.589+0000 7fdbe94c7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.591 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.589+0000 7fdbe94c7500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:44:31.591 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.589+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.591 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.590+0000 7fdbe94c7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.591 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.590+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.593 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.590+0000 7fdbe94c7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:31.593 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.590+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.593 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.590+0000 7fdbe94c7500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:44:31.593 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.590+0000 7fdbe94c7500 20 zonegroup default 2023-09-22T13:44:31.593 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.590+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.593 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.591+0000 7fdbe94c7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.593 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.591+0000 7fdbe94c7500 10 Cannot find current period zone using local zone 2023-09-22T13:44:31.593 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.591+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.593 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.591+0000 7fdbe94c7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.593 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.591+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.593 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.591+0000 7fdbe94c7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:44:31.594 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.591+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.594 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.591+0000 7fdbe94c7500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:44:31.594 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.591+0000 7fdbe94c7500 20 zone default found 2023-09-22T13:44:31.594 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.591+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.594 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.592+0000 7fdbe94c7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.594 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.592+0000 7fdbe94c7500 20 rados->read ofs=0 len=0 2023-09-22T13:44:31.594 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.592+0000 7fdbe94c7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:44:31.594 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.592+0000 7fdbe94c7500 20 started sync module instance, tier type = 2023-09-22T13:44:31.594 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.592+0000 7fdbe94c7500 20 started zone id=f54d190c-30d0-4b87-88c4-5bcc7de96de4 (name=default) with tier type = 2023-09-22T13:44:31.602 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.601+0000 7fdbe94c7500 20 add_watcher() i=0 2023-09-22T13:44:31.602 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.601+0000 7fdbe94c7500 20 add_watcher() i=1 2023-09-22T13:44:31.602 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.601+0000 7fdbe94c7500 20 add_watcher() i=2 2023-09-22T13:44:31.602 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.601+0000 7fdbe94c7500 20 add_watcher() i=3 2023-09-22T13:44:31.602 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.601+0000 7fdbe94c7500 20 add_watcher() i=4 2023-09-22T13:44:31.602 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.601+0000 7fdbe94c7500 20 add_watcher() i=5 2023-09-22T13:44:31.602 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.601+0000 7fdbe94c7500 20 add_watcher() i=6 2023-09-22T13:44:31.603 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.602+0000 7fdbd49a9700 0 -- 172.21.15.69:0/1243054943 >> [v2:172.21.15.69:6800/145165,v1:172.21.15.69:6801/145165] conn(0x7fdbc801f910 msgr2=0x7fdbc801d3e0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:44:31.603 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.602+0000 7fdbe94c7500 20 add_watcher() i=7 2023-09-22T13:44:31.603 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.602+0000 7fdbe94c7500 2 all 8 watchers are set, enabling cache 2023-09-22T13:44:31.604 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.603+0000 7fdba5fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:44:31.604 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.603+0000 7fdbe94c7500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:44:31.604 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.603+0000 7fdbe94c7500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:44:31.607 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.606+0000 7fdbd49a9700 0 -- 172.21.15.69:0/3097685800 >> [v2:172.21.15.69:3301/0,v1:172.21.15.69:6790/0] conn(0x561981961210 msgr2=0x561981967240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:44:31.807 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.807+0000 7fdbd49a9700 0 --2- 172.21.15.69:0/3097685800 >> [v2:172.21.15.69:3301/0,v1:172.21.15.69:6790/0] conn(0x561981961210 0x561981967240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:44:31.810 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.809+0000 7fdbe94c7500 5 note: GC not initialized 2023-09-22T13:44:31.811 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.810+0000 7fdb1cff9700 20 reqs_thread_entry: start 2023-09-22T13:44:31.811 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.810+0000 7fdb127fc700 20 reqs_thread_entry: start 2023-09-22T13:44:31.812 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.810+0000 7fdbe94c7500 20 init_complete bucket index max shards: 11 2023-09-22T13:44:31.812 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.811+0000 7fdbe94c7500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:44:31.812 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.811+0000 7fdb10ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:45:02 2023 2023-09-22T13:44:31.812 INFO:teuthology.orchestra.run.smithi069.stderr: 2023-09-22T13:44:31.812 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.811+0000 7fdaf7fff700 20 reqs_thread_entry: start 2023-09-22T13:44:31.812 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.811+0000 7fdbe94c7500 20 get_system_obj_state: rctx=0x7ffc53a77768 obj=default.rgw.meta:users.uid:sysuser.client.0 state=0x561981800810 s->prefetch_data=0 2023-09-22T13:44:31.812 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.811+0000 7fdbe94c7500 10 cache get: name=default.rgw.meta+users.uid+sysuser.client.0 : miss 2023-09-22T13:44:31.813 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.811+0000 7fdbe94c7500 10 cache put: name=default.rgw.meta+users.uid+sysuser.client.0 info.flags=0x0 2023-09-22T13:44:31.813 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.811+0000 7fdbe94c7500 10 adding default.rgw.meta+users.uid+sysuser.client.0 to cache LRU end 2023-09-22T13:44:31.813 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.811+0000 7fdbe94c7500 20 get_system_obj_state: rctx=0x7ffc53a77778 obj=default.rgw.meta:users.email:sysuser.client.0+test@test.test state=0x561981800810 s->prefetch_data=0 2023-09-22T13:44:31.813 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.811+0000 7fdbe94c7500 10 cache get: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test : miss 2023-09-22T13:44:31.813 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.812+0000 7fdbe94c7500 10 cache put: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test info.flags=0x0 2023-09-22T13:44:31.814 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.812+0000 7fdbe94c7500 10 adding default.rgw.meta+users.email+sysuser.client.0+test@test.test to cache LRU end 2023-09-22T13:44:31.814 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.812+0000 7fdbe94c7500 20 get_system_obj_state: rctx=0x7ffc53a77778 obj=default.rgw.meta:users.keys:GRVTXEKBJSFREVTTDYYI state=0x561981800810 s->prefetch_data=0 2023-09-22T13:44:31.814 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.812+0000 7fdbe94c7500 10 cache get: name=default.rgw.meta+users.keys+GRVTXEKBJSFREVTTDYYI : miss 2023-09-22T13:44:31.814 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.812+0000 7fdbe94c7500 10 cache put: name=default.rgw.meta+users.keys+GRVTXEKBJSFREVTTDYYI info.flags=0x0 2023-09-22T13:44:31.814 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.812+0000 7fdbe94c7500 10 adding default.rgw.meta+users.keys+GRVTXEKBJSFREVTTDYYI to cache LRU end 2023-09-22T13:44:31.814 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.812+0000 7fdbe94c7500 20 get_system_obj_state: rctx=0x7ffc53a77078 obj=default.rgw.meta:users.keys:GRVTXEKBJSFREVTTDYYI state=0x561981800810 s->prefetch_data=0 2023-09-22T13:44:31.814 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.812+0000 7fdbe94c7500 10 cache get: name=default.rgw.meta+users.keys+GRVTXEKBJSFREVTTDYYI : hit (negative entry) 2023-09-22T13:44:31.814 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.812+0000 7fdbe94c7500 20 get_system_obj_state: rctx=0x7ffc53a773b8 obj=default.rgw.meta:users.keys:GRVTXEKBJSFREVTTDYYI state=0x561981800810 s->prefetch_data=0 2023-09-22T13:44:31.814 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.812+0000 7fdbe94c7500 10 cache get: name=default.rgw.meta+users.keys+GRVTXEKBJSFREVTTDYYI : hit (negative entry) 2023-09-22T13:44:31.815 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.814+0000 7fdbe94c7500 10 cache put: name=default.rgw.meta+users.uid+sysuser.client.0 info.flags=0x17 2023-09-22T13:44:31.815 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.814+0000 7fdbe94c7500 10 moving default.rgw.meta+users.uid+sysuser.client.0 to cache LRU end 2023-09-22T13:44:31.815 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.814+0000 7fdbe94c7500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.uid:sysuser.client.0, ofs0, ns] 2023-09-22T13:44:31.815 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.814+0000 7fdbce7fc700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1005022347272 cookie 94667548194624 notifier 5661 bl.length()=641 2023-09-22T13:44:31.815 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.814+0000 7fdbce7fc700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+sysuser.client.0 info.flags=0x17 2023-09-22T13:44:31.815 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.814+0000 7fdbce7fc700 10 rgw watcher librados: moving default.rgw.meta+users.uid+sysuser.client.0 to cache LRU end 2023-09-22T13:44:31.817 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.816+0000 7fdbe94c7500 10 cache put: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test info.flags=0x7 2023-09-22T13:44:31.817 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.816+0000 7fdbe94c7500 10 moving default.rgw.meta+users.email+sysuser.client.0+test@test.test to cache LRU end 2023-09-22T13:44:31.817 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.816+0000 7fdbe94c7500 10 distributing notification oid=default.rgw.control:notify.4 cni=[op: 0, obj: default.rgw.meta:users.email:sysuser.client.0+test@test.test, ofs0, ns] 2023-09-22T13:44:31.817 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.816+0000 7fdbcdffb700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1005022347305 cookie 94667548201472 notifier 5661 bl.length()=202 2023-09-22T13:44:31.817 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.816+0000 7fdbcdffb700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test info.flags=0x7 2023-09-22T13:44:31.817 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.816+0000 7fdbcdffb700 10 rgw watcher librados: moving default.rgw.meta+users.email+sysuser.client.0+test@test.test to cache LRU end 2023-09-22T13:44:31.819 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.818+0000 7fdbe94c7500 10 cache put: name=default.rgw.meta+users.keys+GRVTXEKBJSFREVTTDYYI info.flags=0x7 2023-09-22T13:44:31.819 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.818+0000 7fdbe94c7500 10 moving default.rgw.meta+users.keys+GRVTXEKBJSFREVTTDYYI to cache LRU end 2023-09-22T13:44:31.819 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.818+0000 7fdbe94c7500 10 distributing notification oid=default.rgw.control:notify.6 cni=[op: 0, obj: default.rgw.meta:users.keys:GRVTXEKBJSFREVTTDYYI, ofs0, ns] 2023-09-22T13:44:31.819 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.818+0000 7fdbcdffb700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1005022347350 cookie 94667548204784 notifier 5661 bl.length()=190 2023-09-22T13:44:31.819 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.818+0000 7fdbcdffb700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+GRVTXEKBJSFREVTTDYYI info.flags=0x7 2023-09-22T13:44:31.819 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.818+0000 7fdbcdffb700 10 rgw watcher librados: moving default.rgw.meta+users.keys+GRVTXEKBJSFREVTTDYYI to cache LRU end 2023-09-22T13:44:31.820 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2023-09-22T13:44:31.820 INFO:teuthology.orchestra.run.smithi069.stdout: "user_id": "sysuser.client.0", 2023-09-22T13:44:31.820 INFO:teuthology.orchestra.run.smithi069.stdout: "display_name": "Mr. sysuser.client.0", 2023-09-22T13:44:31.820 INFO:teuthology.orchestra.run.smithi069.stdout: "email": "sysuser.client.0+test@test.test", 2023-09-22T13:44:31.820 INFO:teuthology.orchestra.run.smithi069.stdout: "suspended": 0, 2023-09-22T13:44:31.820 INFO:teuthology.orchestra.run.smithi069.stdout: "max_buckets": 1000, 2023-09-22T13:44:31.820 INFO:teuthology.orchestra.run.smithi069.stdout: "subusers": [], 2023-09-22T13:44:31.820 INFO:teuthology.orchestra.run.smithi069.stdout: "keys": [ 2023-09-22T13:44:31.820 INFO:teuthology.orchestra.run.smithi069.stdout: { 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: "user": "sysuser.client.0", 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: "access_key": "GRVTXEKBJSFREVTTDYYI", 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: "secret_key": "JhuFkyWULldI4j74AAvGNc1Isbx6dJmGFiSxH5/TyUyRZwrLuBp6VA==" 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: } 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: "swift_keys": [], 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: "caps": [], 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: "op_mask": "read, write, delete", 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: "system": "true", 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: "default_placement": "", 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: "default_storage_class": "", 2023-09-22T13:44:31.821 INFO:teuthology.orchestra.run.smithi069.stdout: "placement_tags": [], 2023-09-22T13:44:31.822 INFO:teuthology.orchestra.run.smithi069.stdout: "bucket_quota": { 2023-09-22T13:44:31.822 INFO:teuthology.orchestra.run.smithi069.stdout: "enabled": false, 2023-09-22T13:44:31.822 INFO:teuthology.orchestra.run.smithi069.stdout: "check_on_raw": false, 2023-09-22T13:44:31.822 INFO:teuthology.orchestra.run.smithi069.stdout: "max_size": -1, 2023-09-22T13:44:31.822 INFO:teuthology.orchestra.run.smithi069.stdout: "max_size_kb": 0, 2023-09-22T13:44:31.822 INFO:teuthology.orchestra.run.smithi069.stdout: "max_objects": -1 2023-09-22T13:44:31.822 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:44:31.822 INFO:teuthology.orchestra.run.smithi069.stdout: "user_quota": { 2023-09-22T13:44:31.822 INFO:teuthology.orchestra.run.smithi069.stdout: "enabled": false, 2023-09-22T13:44:31.822 INFO:teuthology.orchestra.run.smithi069.stdout: "check_on_raw": false, 2023-09-22T13:44:31.822 INFO:teuthology.orchestra.run.smithi069.stdout: "max_size": -1, 2023-09-22T13:44:31.823 INFO:teuthology.orchestra.run.smithi069.stdout: "max_size_kb": 0, 2023-09-22T13:44:31.823 INFO:teuthology.orchestra.run.smithi069.stdout: "max_objects": -1 2023-09-22T13:44:31.823 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2023-09-22T13:44:31.823 INFO:teuthology.orchestra.run.smithi069.stdout: "temp_url_keys": [], 2023-09-22T13:44:31.823 INFO:teuthology.orchestra.run.smithi069.stdout: "type": "rgw", 2023-09-22T13:44:31.823 INFO:teuthology.orchestra.run.smithi069.stdout: "mfa_ids": [] 2023-09-22T13:44:31.823 INFO:teuthology.orchestra.run.smithi069.stdout:} 2023-09-22T13:44:31.823 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:44:31.824 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.820+0000 7fdbe94c7500 20 remove_watcher() i=0 2023-09-22T13:44:31.824 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.820+0000 7fdbe94c7500 2 removed watcher, disabling cache 2023-09-22T13:44:31.824 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.821+0000 7fdbe94c7500 20 remove_watcher() i=1 2023-09-22T13:44:31.824 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.822+0000 7fdbe94c7500 20 remove_watcher() i=2 2023-09-22T13:44:31.824 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.824+0000 7fdbe94c7500 20 remove_watcher() i=3 2023-09-22T13:44:31.825 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.825+0000 7fdbe94c7500 20 remove_watcher() i=4 2023-09-22T13:44:31.826 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.826+0000 7fdbe94c7500 20 remove_watcher() i=5 2023-09-22T13:44:31.827 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.827+0000 7fdbe94c7500 20 remove_watcher() i=6 2023-09-22T13:44:31.828 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.828+0000 7fdbe94c7500 20 remove_watcher() i=7 2023-09-22T13:44:31.829 INFO:teuthology.orchestra.run.smithi069.stderr:2023-09-22T13:44:31.828+0000 7fdba5fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:44:31.834 INFO:tasks.ragweed:Configuring ragweed... 2023-09-22T13:44:31.834 DEBUG:teuthology.orchestra.run.smithi069:> cd /home/ubuntu/cephtest/ragweed && ./bootstrap 2023-09-22T13:44:31.852 INFO:teuthology.orchestra.run.smithi069.stderr:+ set -e 2023-09-22T13:44:31.853 INFO:teuthology.orchestra.run.smithi069.stderr:+ '[' -f /etc/debian_version ']' 2023-09-22T13:44:31.853 INFO:teuthology.orchestra.run.smithi069.stderr:+ '[' -f /etc/redhat-release ']' 2023-09-22T13:44:31.853 INFO:teuthology.orchestra.run.smithi069.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:44:31.853 INFO:teuthology.orchestra.run.smithi069.stderr:++ rpm -qa python3-pip 2023-09-22T13:44:33.053 INFO:teuthology.orchestra.run.smithi069.stderr:+ '[' python3-pip-9.0.3-22.el8.noarch == '' ']' 2023-09-22T13:44:33.370 INFO:teuthology.orchestra.run.smithi069.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:44:33.370 INFO:teuthology.orchestra.run.smithi069.stderr:++ rpm -qa python3-virtualenv 2023-09-22T13:44:34.190 INFO:teuthology.orchestra.run.smithi069.stderr:+ '[' python3-virtualenv-15.1.0-21.module_el8.5.0+895+a459eca8.noarch == '' ']' 2023-09-22T13:44:34.190 INFO:teuthology.orchestra.run.smithi069.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:44:34.190 INFO:teuthology.orchestra.run.smithi069.stderr:++ rpm -qa python3-devel 2023-09-22T13:44:35.327 INFO:teuthology.orchestra.run.smithi069.stderr:+ '[' '' == '' ']' 2023-09-22T13:44:35.327 INFO:teuthology.orchestra.run.smithi069.stderr:+ missing=python3-devel 2023-09-22T13:44:35.327 INFO:teuthology.orchestra.run.smithi069.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:44:35.327 INFO:teuthology.orchestra.run.smithi069.stderr:++ rpm -qa libevent-devel 2023-09-22T13:44:36.463 INFO:teuthology.orchestra.run.smithi069.stderr:+ '[' '' == '' ']' 2023-09-22T13:44:36.463 INFO:teuthology.orchestra.run.smithi069.stderr:+ missing='python3-devel libevent-devel' 2023-09-22T13:44:36.464 INFO:teuthology.orchestra.run.smithi069.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:44:36.464 INFO:teuthology.orchestra.run.smithi069.stderr:++ rpm -qa libxml2-devel 2023-09-22T13:44:37.600 INFO:teuthology.orchestra.run.smithi069.stderr:+ '[' '' == '' ']' 2023-09-22T13:44:37.600 INFO:teuthology.orchestra.run.smithi069.stderr:+ missing='python3-devel libevent-devel libxml2-devel' 2023-09-22T13:44:37.600 INFO:teuthology.orchestra.run.smithi069.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:44:37.600 INFO:teuthology.orchestra.run.smithi069.stderr:++ rpm -qa libxslt-devel 2023-09-22T13:44:38.747 INFO:teuthology.orchestra.run.smithi069.stderr:+ '[' '' == '' ']' 2023-09-22T13:44:38.747 INFO:teuthology.orchestra.run.smithi069.stderr:+ missing='python3-devel libevent-devel libxml2-devel libxslt-devel' 2023-09-22T13:44:38.747 INFO:teuthology.orchestra.run.smithi069.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:44:38.747 INFO:teuthology.orchestra.run.smithi069.stderr:++ rpm -qa zlib-devel 2023-09-22T13:44:39.887 INFO:teuthology.orchestra.run.smithi069.stderr:+ '[' '' == '' ']' 2023-09-22T13:44:39.887 INFO:teuthology.orchestra.run.smithi069.stderr:+ missing='python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel' 2023-09-22T13:44:39.887 INFO:teuthology.orchestra.run.smithi069.stderr:+ '[' -n 'python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel' ']' 2023-09-22T13:44:39.887 INFO:teuthology.orchestra.run.smithi069.stderr:+ echo './bootstrap: missing required RPM packages. Installing via sudo.' 2023-09-22T13:44:39.887 INFO:teuthology.orchestra.run.smithi069.stderr:./bootstrap: missing required RPM packages. Installing via sudo. 2023-09-22T13:44:39.887 INFO:teuthology.orchestra.run.smithi069.stderr:+ sudo yum -y install python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:44:40.738 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:38:34 ago on Fri 22 Sep 2023 01:06:06 PM UTC. 2023-09-22T13:44:42.201 INFO:teuthology.orchestra.run.smithi069.stdout:Package python36-devel-3.6.8-38.module_el8.5.0+895+a459eca8.x86_64 is already installed. 2023-09-22T13:44:42.393 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2023-09-22T13:44:42.395 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:44:42.395 INFO:teuthology.orchestra.run.smithi069.stdout: Package Arch Version Repository Size 2023-09-22T13:44:42.395 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:44:42.395 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2023-09-22T13:44:42.395 INFO:teuthology.orchestra.run.smithi069.stdout: libevent-devel x86_64 2.1.8-5.el8 appstream 104 k 2023-09-22T13:44:42.395 INFO:teuthology.orchestra.run.smithi069.stdout: libxml2-devel x86_64 2.9.7-16.el8 appstream 1.0 M 2023-09-22T13:44:42.395 INFO:teuthology.orchestra.run.smithi069.stdout: libxslt-devel x86_64 1.1.32-6.el8 appstream 323 k 2023-09-22T13:44:42.395 INFO:teuthology.orchestra.run.smithi069.stdout: zlib-devel x86_64 1.2.11-25.el8 baseos 59 k 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout:Upgrading: 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout: zlib x86_64 1.2.11-25.el8 baseos 103 k 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout: cmake-filesystem x86_64 3.20.2-5.el8 appstream 45 k 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout: libgcrypt-devel x86_64 1.8.5-7.el8 baseos 150 k 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout: libgpg-error-devel x86_64 1.31-1.el8 baseos 64 k 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout: xz-devel x86_64 5.2.4-4.el8 baseos 63 k 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout:Install 8 Packages 2023-09-22T13:44:42.396 INFO:teuthology.orchestra.run.smithi069.stdout:Upgrade 1 Package 2023-09-22T13:44:42.397 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:44:42.397 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 1.9 M 2023-09-22T13:44:42.397 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2023-09-22T13:44:43.257 INFO:teuthology.orchestra.run.smithi069.stdout:(1/9): cmake-filesystem-3.20.2-5.el8.x86_64.rpm 91 kB/s | 45 kB 00:00 2023-09-22T13:44:45.257 INFO:teuthology.orchestra.run.smithi069.stdout:(2/9): libevent-devel-2.1.8-5.el8.x86_64.rpm 42 kB/s | 104 kB 00:02 2023-09-22T13:44:46.014 INFO:teuthology.orchestra.run.smithi069.stdout:(3/9): libgcrypt-devel-1.8.5-7.el8.x86_64.rpm 198 kB/s | 150 kB 00:00 2023-09-22T13:44:46.325 INFO:teuthology.orchestra.run.smithi069.stdout:(4/9): libgpg-error-devel-1.31-1.el8.x86_64.rpm 205 kB/s | 64 kB 00:00 2023-09-22T13:44:46.683 INFO:teuthology.orchestra.run.smithi069.stdout:(5/9): xz-devel-5.2.4-4.el8.x86_64.rpm 175 kB/s | 63 kB 00:00 2023-09-22T13:44:46.984 INFO:teuthology.orchestra.run.smithi069.stdout:(6/9): zlib-devel-1.2.11-25.el8.x86_64.rpm 195 kB/s | 59 kB 00:00 2023-09-22T13:44:47.401 INFO:teuthology.orchestra.run.smithi069.stdout:(7/9): zlib-1.2.11-25.el8.x86_64.rpm 248 kB/s | 103 kB 00:00 2023-09-22T13:44:54.067 INFO:teuthology.orchestra.run.smithi069.stdout:(8/9): libxslt-devel-1.1.32-6.el8.x86_64.rpm 30 kB/s | 323 kB 00:10 2023-09-22T13:45:26.814 INFO:teuthology.orchestra.run.smithi069.stdout:(9/9): libxml2-devel-2.9.7-16.el8.x86_64.rpm 24 kB/s | 1.0 MB 00:44 2023-09-22T13:45:26.815 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:45:26.815 INFO:teuthology.orchestra.run.smithi069.stdout:Total 44 kB/s | 1.9 MB 00:44 2023-09-22T13:45:26.913 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2023-09-22T13:45:27.055 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2023-09-22T13:45:27.055 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2023-09-22T13:45:27.180 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2023-09-22T13:45:27.183 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2023-09-22T13:45:27.545 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2023-09-22T13:45:27.692 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libgpg-error-devel-1.31-1.el8.x86_64 1/10 2023-09-22T13:45:27.720 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: libgpg-error-devel-1.31-1.el8.x86_64 1/10 2023-09-22T13:45:27.917 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libgcrypt-devel-1.8.5-7.el8.x86_64 2/10 2023-09-22T13:45:27.934 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: libgcrypt-devel-1.8.5-7.el8.x86_64 2/10 2023-09-22T13:45:28.122 INFO:teuthology.orchestra.run.smithi069.stdout: Upgrading : zlib-1.2.11-25.el8.x86_64 3/10 2023-09-22T13:45:28.296 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : zlib-devel-1.2.11-25.el8.x86_64 4/10 2023-09-22T13:45:28.465 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : xz-devel-5.2.4-4.el8.x86_64 5/10 2023-09-22T13:45:28.902 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : cmake-filesystem-3.20.2-5.el8.x86_64 6/10 2023-09-22T13:45:29.233 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libxml2-devel-2.9.7-16.el8.x86_64 7/10 2023-09-22T13:45:29.479 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libxslt-devel-1.1.32-6.el8.x86_64 8/10 2023-09-22T13:45:29.642 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : libevent-devel-2.1.8-5.el8.x86_64 9/10 2023-09-22T13:45:29.807 INFO:teuthology.orchestra.run.smithi069.stdout: Cleanup : zlib-1.2.11-21.el8.x86_64 10/10 2023-09-22T13:45:30.112 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: zlib-1.2.11-21.el8.x86_64 10/10 2023-09-22T13:45:30.112 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : cmake-filesystem-3.20.2-5.el8.x86_64 1/10 2023-09-22T13:45:30.112 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libevent-devel-2.1.8-5.el8.x86_64 2/10 2023-09-22T13:45:30.112 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libxml2-devel-2.9.7-16.el8.x86_64 3/10 2023-09-22T13:45:30.112 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libxslt-devel-1.1.32-6.el8.x86_64 4/10 2023-09-22T13:45:30.112 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libgcrypt-devel-1.8.5-7.el8.x86_64 5/10 2023-09-22T13:45:30.112 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : libgpg-error-devel-1.31-1.el8.x86_64 6/10 2023-09-22T13:45:30.112 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : xz-devel-5.2.4-4.el8.x86_64 7/10 2023-09-22T13:45:30.113 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : zlib-devel-1.2.11-25.el8.x86_64 8/10 2023-09-22T13:45:30.113 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : zlib-1.2.11-25.el8.x86_64 9/10 2023-09-22T13:45:30.796 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : zlib-1.2.11-21.el8.x86_64 10/10 2023-09-22T13:45:30.796 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:30.797 INFO:teuthology.orchestra.run.smithi069.stdout:Upgraded: 2023-09-22T13:45:30.797 INFO:teuthology.orchestra.run.smithi069.stdout: zlib-1.2.11-25.el8.x86_64 2023-09-22T13:45:30.797 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2023-09-22T13:45:30.797 INFO:teuthology.orchestra.run.smithi069.stdout: cmake-filesystem-3.20.2-5.el8.x86_64 libevent-devel-2.1.8-5.el8.x86_64 2023-09-22T13:45:30.797 INFO:teuthology.orchestra.run.smithi069.stdout: libgcrypt-devel-1.8.5-7.el8.x86_64 libgpg-error-devel-1.31-1.el8.x86_64 2023-09-22T13:45:30.797 INFO:teuthology.orchestra.run.smithi069.stdout: libxml2-devel-2.9.7-16.el8.x86_64 libxslt-devel-1.1.32-6.el8.x86_64 2023-09-22T13:45:30.797 INFO:teuthology.orchestra.run.smithi069.stdout: xz-devel-5.2.4-4.el8.x86_64 zlib-devel-1.2.11-25.el8.x86_64 2023-09-22T13:45:30.798 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:30.798 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2023-09-22T13:45:31.005 INFO:teuthology.orchestra.run.smithi069.stderr:+ sudo pip3 install --upgrade --trusted-host apt-mirror.front.sepia.ceph.com setuptools cffi cachecontrol 2023-09-22T13:45:31.322 INFO:teuthology.orchestra.run.smithi069.stderr:WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead. 2023-09-22T13:45:31.867 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting setuptools 2023-09-22T13:45:32.235 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading https://files.pythonhosted.org/packages/b0/3a/88b210db68e56854d0bcf4b38e165e03be377e13907746f825790f3df5bf/setuptools-59.6.0-py3-none-any.whl (952kB) 2023-09-22T13:45:32.772 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting cffi 2023-09-22T13:45:32.996 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading https://files.pythonhosted.org/packages/3a/12/d6066828014b9ccb2bbb8e1d9dc28872d20669b65aeb4a86806a0757813f/cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (402kB) 2023-09-22T13:45:33.094 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting cachecontrol 2023-09-22T13:45:33.224 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading https://files.pythonhosted.org/packages/72/a2/28e0ef082f7d78253aded97933e1d7b94bab3c5be366e8afd6513de4028e/CacheControl-0.12.14-py2.py3-none-any.whl 2023-09-22T13:45:33.392 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting pycparser (from cffi) 2023-09-22T13:45:33.417 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading https://files.pythonhosted.org/packages/62/d5/5f610ebe421e85889f2e55e33b7f9a6795bd982198517d912eb1c76e1a53/pycparser-2.21-py2.py3-none-any.whl (118kB) 2023-09-22T13:45:33.563 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting requests (from cachecontrol) 2023-09-22T13:45:33.631 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading https://files.pythonhosted.org/packages/2d/61/08076519c80041bc0ffa1a8af0cbd3bf3e2b62af10435d269a9d0f40564d/requests-2.27.1-py2.py3-none-any.whl (63kB) 2023-09-22T13:45:33.727 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting msgpack>=0.5.2 (from cachecontrol) 2023-09-22T13:45:33.941 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading https://files.pythonhosted.org/packages/dc/a1/eba11a0d4b764bc62966a565b470f8c6f38242723ba3057e9b5098678c30/msgpack-1.0.5.tar.gz (127kB) 2023-09-22T13:45:34.486 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting urllib3<1.27,>=1.21.1 (from requests->cachecontrol) 2023-09-22T13:45:34.555 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading https://files.pythonhosted.org/packages/c5/05/c214b32d21c0b465506f95c4f28ccbcba15022e000b043b72b3df7728471/urllib3-1.26.16-py2.py3-none-any.whl (143kB) 2023-09-22T13:45:34.684 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting idna<4,>=2.5; python_version >= "3" (from requests->cachecontrol) 2023-09-22T13:45:34.715 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl (61kB) 2023-09-22T13:45:34.785 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting charset-normalizer~=2.0.0; python_version >= "3" (from requests->cachecontrol) 2023-09-22T13:45:35.014 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading https://files.pythonhosted.org/packages/06/b3/24afc8868eba069a7f03650ac750a778862dc34941a4bebeb58706715726/charset_normalizer-2.0.12-py3-none-any.whl 2023-09-22T13:45:35.215 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting certifi>=2017.4.17 (from requests->cachecontrol) 2023-09-22T13:45:35.259 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl (158kB) 2023-09-22T13:45:35.276 INFO:teuthology.orchestra.run.smithi069.stdout:Installing collected packages: setuptools, pycparser, cffi, urllib3, idna, charset-normalizer, certifi, requests, msgpack, cachecontrol 2023-09-22T13:45:35.791 INFO:teuthology.orchestra.run.smithi069.stdout: Running setup.py install for msgpack: started 2023-09-22T13:45:36.082 INFO:teuthology.orchestra.run.smithi069.stdout: Running setup.py install for msgpack: finished with status 'done' 2023-09-22T13:45:36.127 INFO:teuthology.orchestra.run.smithi069.stdout:Successfully installed cachecontrol-0.12.14 certifi-2023.7.22 cffi-1.15.1 charset-normalizer-2.0.12 idna-3.4 msgpack-1.0.5 pycparser-2.21 requests-2.27.1 setuptools-59.6.0 urllib3-1.26.16 2023-09-22T13:45:36.253 INFO:teuthology.orchestra.run.smithi069.stderr:+ virtualenv -p python3 --system-site-packages --download virtualenv 2023-09-22T13:45:36.669 INFO:teuthology.orchestra.run.smithi069.stdout:Using base prefix '/usr' 2023-09-22T13:45:36.743 INFO:teuthology.orchestra.run.smithi069.stdout:New python executable in /home/ubuntu/cephtest/ragweed/virtualenv/bin/python3 2023-09-22T13:45:36.743 INFO:teuthology.orchestra.run.smithi069.stdout:Also creating executable in /home/ubuntu/cephtest/ragweed/virtualenv/bin/python 2023-09-22T13:45:40.121 INFO:teuthology.orchestra.run.smithi069.stdout:Installing setuptools, pip, wheel...done. 2023-09-22T13:45:40.130 INFO:teuthology.orchestra.run.smithi069.stdout:Running virtualenv with interpreter /usr/bin/python3 2023-09-22T13:45:40.143 INFO:teuthology.orchestra.run.smithi069.stderr:+ ./virtualenv/bin/pip install --upgrade pip 2023-09-22T13:45:40.515 INFO:teuthology.orchestra.run.smithi069.stdout:Requirement already satisfied: pip in ./virtualenv/lib/python3.6/site-packages (21.3.1) 2023-09-22T13:45:41.141 INFO:teuthology.orchestra.run.smithi069.stderr:+ ./virtualenv/bin/pip install six 2023-09-22T13:45:41.506 INFO:teuthology.orchestra.run.smithi069.stdout:Requirement already satisfied: six in /usr/lib/python3.6/site-packages (1.11.0) 2023-09-22T13:45:41.691 INFO:teuthology.orchestra.run.smithi069.stderr:+ ./virtualenv/bin/pip install -I nose 2023-09-22T13:45:42.180 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting nose 2023-09-22T13:45:42.249 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading nose-1.3.7-py3-none-any.whl (154 kB) 2023-09-22T13:45:42.454 INFO:teuthology.orchestra.run.smithi069.stdout:Installing collected packages: nose 2023-09-22T13:45:42.556 INFO:teuthology.orchestra.run.smithi069.stdout:Successfully installed nose-1.3.7 2023-09-22T13:45:42.611 INFO:teuthology.orchestra.run.smithi069.stderr:+ ./virtualenv/bin/pip install setuptools 2023-09-22T13:45:42.982 INFO:teuthology.orchestra.run.smithi069.stdout:Requirement already satisfied: setuptools in ./virtualenv/lib/python3.6/site-packages (59.6.0) 2023-09-22T13:45:43.163 INFO:teuthology.orchestra.run.smithi069.stderr:+ ./virtualenv/bin/pip install -U -r requirements.txt 2023-09-22T13:45:43.532 INFO:teuthology.orchestra.run.smithi069.stdout:Requirement already satisfied: PyYAML in /usr/lib64/python3.6/site-packages (from -r requirements.txt (line 1)) (3.12) 2023-09-22T13:45:43.761 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting PyYAML 2023-09-22T13:45:43.816 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (677 kB) 2023-09-22T13:45:43.917 INFO:teuthology.orchestra.run.smithi069.stdout:Requirement already satisfied: nose>=1.0.0 in ./virtualenv/lib/python3.6/site-packages (from -r requirements.txt (line 2)) (1.3.7) 2023-09-22T13:45:44.041 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting boto!=2.46.0,>=2.6.0 2023-09-22T13:45:44.061 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading boto-2.49.0-py2.py3-none-any.whl (1.4 MB) 2023-09-22T13:45:44.170 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting munch>=1.0.0 2023-09-22T13:45:44.188 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading munch-4.0.0-py2.py3-none-any.whl (9.9 kB) 2023-09-22T13:45:44.630 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting gevent>=1.0 2023-09-22T13:45:44.651 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading gevent-22.10.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.8 MB) 2023-09-22T13:45:44.877 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting httplib2 2023-09-22T13:45:44.894 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading httplib2-0.22.0-py3-none-any.whl (96 kB) 2023-09-22T13:45:44.919 INFO:teuthology.orchestra.run.smithi069.stdout:Requirement already satisfied: lxml in /usr/lib64/python3.6/site-packages (from -r requirements.txt (line 7)) (4.2.3) 2023-09-22T13:45:45.478 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting lxml 2023-09-22T13:45:45.505 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading lxml-4.9.3-cp36-cp36m-manylinux_2_28_x86_64.whl (7.4 MB) 2023-09-22T13:45:45.885 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting importlib-metadata>=1.7.0 2023-09-22T13:45:45.902 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading importlib_metadata-4.8.3-py3-none-any.whl (17 kB) 2023-09-22T13:45:46.321 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting zope.interface 2023-09-22T13:45:46.339 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading zope.interface-5.5.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (253 kB) 2023-09-22T13:45:46.373 INFO:teuthology.orchestra.run.smithi069.stdout:Requirement already satisfied: setuptools in ./virtualenv/lib/python3.6/site-packages (from gevent>=1.0->-r requirements.txt (line 5)) (59.6.0) 2023-09-22T13:45:46.419 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting zope.event 2023-09-22T13:45:46.438 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading zope.event-4.6-py2.py3-none-any.whl (6.8 kB) 2023-09-22T13:45:46.905 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting greenlet>=2.0.0 2023-09-22T13:45:46.925 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading greenlet-2.0.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (564 kB) 2023-09-22T13:45:47.076 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 2023-09-22T13:45:47.114 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading pyparsing-3.1.1-py3-none-any.whl (103 kB) 2023-09-22T13:45:47.267 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting typing-extensions>=3.6.4 2023-09-22T13:45:47.285 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading typing_extensions-4.1.1-py3-none-any.whl (26 kB) 2023-09-22T13:45:47.397 INFO:teuthology.orchestra.run.smithi069.stdout:Collecting zipp>=0.5 2023-09-22T13:45:47.422 INFO:teuthology.orchestra.run.smithi069.stdout: Downloading zipp-3.6.0-py3-none-any.whl (5.3 kB) 2023-09-22T13:45:47.723 INFO:teuthology.orchestra.run.smithi069.stdout:Installing collected packages: zipp, typing-extensions, zope.interface, zope.event, pyparsing, importlib-metadata, greenlet, PyYAML, munch, lxml, httplib2, gevent, boto 2023-09-22T13:45:48.003 INFO:teuthology.orchestra.run.smithi069.stdout: Attempting uninstall: pyparsing 2023-09-22T13:45:48.003 INFO:teuthology.orchestra.run.smithi069.stdout: Found existing installation: pyparsing 2.1.10 2023-09-22T13:45:48.004 INFO:teuthology.orchestra.run.smithi069.stdout: Not uninstalling pyparsing at /usr/lib/python3.6/site-packages, outside environment /home/ubuntu/cephtest/ragweed/virtualenv 2023-09-22T13:45:48.004 INFO:teuthology.orchestra.run.smithi069.stdout: Can't uninstall 'pyparsing'. No files were found to uninstall. 2023-09-22T13:45:48.188 INFO:teuthology.orchestra.run.smithi069.stdout: Attempting uninstall: PyYAML 2023-09-22T13:45:48.189 INFO:teuthology.orchestra.run.smithi069.stdout: Found existing installation: PyYAML 3.12 2023-09-22T13:45:48.189 INFO:teuthology.orchestra.run.smithi069.stdout: Not uninstalling pyyaml at /usr/lib64/python3.6/site-packages, outside environment /home/ubuntu/cephtest/ragweed/virtualenv 2023-09-22T13:45:48.189 INFO:teuthology.orchestra.run.smithi069.stdout: Can't uninstall 'PyYAML'. No files were found to uninstall. 2023-09-22T13:45:48.331 INFO:teuthology.orchestra.run.smithi069.stdout: Attempting uninstall: lxml 2023-09-22T13:45:48.331 INFO:teuthology.orchestra.run.smithi069.stdout: Found existing installation: lxml 4.2.3 2023-09-22T13:45:48.331 INFO:teuthology.orchestra.run.smithi069.stdout: Not uninstalling lxml at /usr/lib64/python3.6/site-packages, outside environment /home/ubuntu/cephtest/ragweed/virtualenv 2023-09-22T13:45:48.331 INFO:teuthology.orchestra.run.smithi069.stdout: Can't uninstall 'lxml'. No files were found to uninstall. 2023-09-22T13:45:49.486 INFO:teuthology.orchestra.run.smithi069.stdout:Successfully installed PyYAML-6.0.1 boto-2.49.0 gevent-22.10.2 greenlet-2.0.2 httplib2-0.22.0 importlib-metadata-4.8.3 lxml-4.9.3 munch-4.0.0 pyparsing-3.1.1 typing-extensions-4.1.1 zipp-3.6.0 zope.event-4.6 zope.interface-5.5.2 2023-09-22T13:45:49.585 INFO:teuthology.orchestra.run.smithi069.stderr:+ ./virtualenv/bin/python setup.py develop 2023-09-22T13:45:49.825 INFO:teuthology.orchestra.run.smithi069.stdout:running develop 2023-09-22T13:45:49.863 INFO:teuthology.orchestra.run.smithi069.stdout:running egg_info 2023-09-22T13:45:49.863 INFO:teuthology.orchestra.run.smithi069.stdout:creating ragweed.egg-info 2023-09-22T13:45:49.863 INFO:teuthology.orchestra.run.smithi069.stdout:writing ragweed.egg-info/PKG-INFO 2023-09-22T13:45:49.864 INFO:teuthology.orchestra.run.smithi069.stdout:writing dependency_links to ragweed.egg-info/dependency_links.txt 2023-09-22T13:45:49.864 INFO:teuthology.orchestra.run.smithi069.stdout:writing requirements to ragweed.egg-info/requires.txt 2023-09-22T13:45:49.864 INFO:teuthology.orchestra.run.smithi069.stdout:writing top-level names to ragweed.egg-info/top_level.txt 2023-09-22T13:45:49.864 INFO:teuthology.orchestra.run.smithi069.stdout:writing manifest file 'ragweed.egg-info/SOURCES.txt' 2023-09-22T13:45:49.864 INFO:teuthology.orchestra.run.smithi069.stdout:reading manifest file 'ragweed.egg-info/SOURCES.txt' 2023-09-22T13:45:49.865 INFO:teuthology.orchestra.run.smithi069.stdout:adding license file 'LICENSE' 2023-09-22T13:45:49.865 INFO:teuthology.orchestra.run.smithi069.stdout:writing manifest file 'ragweed.egg-info/SOURCES.txt' 2023-09-22T13:45:49.866 INFO:teuthology.orchestra.run.smithi069.stdout:running build_ext 2023-09-22T13:45:49.867 INFO:teuthology.orchestra.run.smithi069.stdout:Creating /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/ragweed.egg-link (link to .) 2023-09-22T13:45:49.867 INFO:teuthology.orchestra.run.smithi069.stdout:Adding ragweed 0.0.1 to easy-install.pth file 2023-09-22T13:45:49.867 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:49.868 INFO:teuthology.orchestra.run.smithi069.stdout:Installed /home/ubuntu/cephtest/ragweed 2023-09-22T13:45:49.868 INFO:teuthology.orchestra.run.smithi069.stdout:Processing dependencies for ragweed==0.0.1 2023-09-22T13:45:49.871 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for isodate>=0.4.4 2023-09-22T13:45:49.871 INFO:teuthology.orchestra.run.smithi069.stdout:Reading https://pypi.org/simple/isodate/ 2023-09-22T13:45:49.954 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading https://files.pythonhosted.org/packages/b6/85/7882d311924cbcfc70b1890780763e36ff0b140c7e51c110fc59a532f087/isodate-0.6.1-py2.py3-none-any.whl#sha256=0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96 2023-09-22T13:45:50.021 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: isodate 0.6.1 2023-09-22T13:45:50.021 INFO:teuthology.orchestra.run.smithi069.stdout:Processing isodate-0.6.1-py2.py3-none-any.whl 2023-09-22T13:45:50.027 INFO:teuthology.orchestra.run.smithi069.stdout:Installing isodate-0.6.1-py2.py3-none-any.whl to /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:45:50.034 INFO:teuthology.orchestra.run.smithi069.stdout:Adding isodate 0.6.1 to easy-install.pth file 2023-09-22T13:45:50.035 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.035 INFO:teuthology.orchestra.run.smithi069.stdout:Installed /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/isodate-0.6.1-py3.6.egg 2023-09-22T13:45:50.139 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for gevent==22.10.2 2023-09-22T13:45:50.139 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: gevent 22.10.2 2023-09-22T13:45:50.139 INFO:teuthology.orchestra.run.smithi069.stdout:Adding gevent 22.10.2 to easy-install.pth file 2023-09-22T13:45:50.139 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.139 INFO:teuthology.orchestra.run.smithi069.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:45:50.140 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for munch==4.0.0 2023-09-22T13:45:50.140 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: munch 4.0.0 2023-09-22T13:45:50.140 INFO:teuthology.orchestra.run.smithi069.stdout:Adding munch 4.0.0 to easy-install.pth file 2023-09-22T13:45:50.140 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.140 INFO:teuthology.orchestra.run.smithi069.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:45:50.141 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for PyYAML==6.0.1 2023-09-22T13:45:50.141 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: PyYAML 6.0.1 2023-09-22T13:45:50.141 INFO:teuthology.orchestra.run.smithi069.stdout:Adding PyYAML 6.0.1 to easy-install.pth file 2023-09-22T13:45:50.141 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.141 INFO:teuthology.orchestra.run.smithi069.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:45:50.142 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for boto==2.49.0 2023-09-22T13:45:50.142 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: boto 2.49.0 2023-09-22T13:45:50.142 INFO:teuthology.orchestra.run.smithi069.stdout:Adding boto 2.49.0 to easy-install.pth file 2023-09-22T13:45:50.142 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.142 INFO:teuthology.orchestra.run.smithi069.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:45:50.143 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for six==1.11.0 2023-09-22T13:45:50.143 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: six 1.11.0 2023-09-22T13:45:50.143 INFO:teuthology.orchestra.run.smithi069.stdout:Adding six 1.11.0 to easy-install.pth file 2023-09-22T13:45:50.143 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.143 INFO:teuthology.orchestra.run.smithi069.stdout:Using /usr/lib/python3.6/site-packages 2023-09-22T13:45:50.144 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for setuptools==59.6.0 2023-09-22T13:45:50.144 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: setuptools 59.6.0 2023-09-22T13:45:50.144 INFO:teuthology.orchestra.run.smithi069.stdout:Adding setuptools 59.6.0 to easy-install.pth file 2023-09-22T13:45:50.144 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.145 INFO:teuthology.orchestra.run.smithi069.stdout:Using /usr/local/lib/python3.6/site-packages 2023-09-22T13:45:50.145 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for greenlet==2.0.2 2023-09-22T13:45:50.145 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: greenlet 2.0.2 2023-09-22T13:45:50.145 INFO:teuthology.orchestra.run.smithi069.stdout:Adding greenlet 2.0.2 to easy-install.pth file 2023-09-22T13:45:50.145 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.146 INFO:teuthology.orchestra.run.smithi069.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:45:50.146 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for zope.event==4.6 2023-09-22T13:45:50.146 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: zope.event 4.6 2023-09-22T13:45:50.147 INFO:teuthology.orchestra.run.smithi069.stdout:Adding zope.event 4.6 to easy-install.pth file 2023-09-22T13:45:50.147 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.147 INFO:teuthology.orchestra.run.smithi069.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:45:50.147 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for zope.interface==5.5.2 2023-09-22T13:45:50.147 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: zope.interface 5.5.2 2023-09-22T13:45:50.148 INFO:teuthology.orchestra.run.smithi069.stdout:Adding zope.interface 5.5.2 to easy-install.pth file 2023-09-22T13:45:50.148 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.148 INFO:teuthology.orchestra.run.smithi069.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:45:50.149 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for importlib-metadata==4.8.3 2023-09-22T13:45:50.149 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: importlib-metadata 4.8.3 2023-09-22T13:45:50.149 INFO:teuthology.orchestra.run.smithi069.stdout:Adding importlib-metadata 4.8.3 to easy-install.pth file 2023-09-22T13:45:50.149 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.149 INFO:teuthology.orchestra.run.smithi069.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:45:50.149 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for zipp==3.6.0 2023-09-22T13:45:50.149 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: zipp 3.6.0 2023-09-22T13:45:50.149 INFO:teuthology.orchestra.run.smithi069.stdout:Adding zipp 3.6.0 to easy-install.pth file 2023-09-22T13:45:50.149 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.149 INFO:teuthology.orchestra.run.smithi069.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:45:50.150 INFO:teuthology.orchestra.run.smithi069.stdout:Searching for typing-extensions==4.1.1 2023-09-22T13:45:50.150 INFO:teuthology.orchestra.run.smithi069.stdout:Best match: typing-extensions 4.1.1 2023-09-22T13:45:50.150 INFO:teuthology.orchestra.run.smithi069.stdout:Adding typing-extensions 4.1.1 to easy-install.pth file 2023-09-22T13:45:50.150 INFO:teuthology.orchestra.run.smithi069.stdout: 2023-09-22T13:45:50.151 INFO:teuthology.orchestra.run.smithi069.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:45:50.151 INFO:teuthology.orchestra.run.smithi069.stdout:Finished processing dependencies for ragweed==0.0.1 2023-09-22T13:45:50.151 INFO:teuthology.orchestra.run.smithi069.stderr:/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/setuptools/command/easy_install.py:159: EasyInstallDeprecationWarning: easy_install command is deprecated. Use build and pip and other standards-based tools. 2023-09-22T13:45:50.151 INFO:teuthology.orchestra.run.smithi069.stderr: EasyInstallDeprecationWarning, 2023-09-22T13:45:50.151 INFO:teuthology.orchestra.run.smithi069.stderr:/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/setuptools/command/install.py:37: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools. 2023-09-22T13:45:50.151 INFO:teuthology.orchestra.run.smithi069.stderr: setuptools.SetuptoolsDeprecationWarning, 2023-09-22T13:45:50.151 INFO:teuthology.orchestra.run.smithi069.stderr:/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/pkg_resources/__init__.py:119: PkgResourcesDeprecationWarning: is an invalid version and will not be supported in a future release 2023-09-22T13:45:50.151 INFO:teuthology.orchestra.run.smithi069.stderr: PkgResourcesDeprecationWarning, 2023-09-22T13:45:50.171 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:45:50.171 DEBUG:teuthology.orchestra.run.smithi069:> dd of=/home/ubuntu/cephtest/archive/ragweed.client.0.conf 2023-09-22T13:45:50.230 INFO:tasks.ragweed:Configuring boto... 2023-09-22T13:45:50.230 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2023-09-22T13:45:50.230 DEBUG:teuthology.orchestra.run.smithi069:> dd of=/home/ubuntu/cephtest/boto.cfg 2023-09-22T13:45:50.288 DEBUG:teuthology.orchestra.run.smithi069:ragweed tests against rgw> RAGWEED_CONF=/home/ubuntu/cephtest/archive/ragweed.client.0.conf RAGWEED_STAGES=prepare,check BOTO_CONFIG=/home/ubuntu/cephtest/boto.cfg /home/ubuntu/cephtest/ragweed/virtualenv/bin/python -m nose -w /home/ubuntu/cephtest/ragweed -v -a '!fails_on_rgw' 2023-09-22T13:45:53.495 INFO:teuthology.orchestra.run.smithi069.stderr:ragweed.tests.tests.r_test_multipart_defer_complete.test ... ok 2023-09-22T13:45:54.541 INFO:teuthology.orchestra.run.smithi069.stderr:ragweed.tests.tests.r_test_multipart_defer_update_complete.test ... ok 2023-09-22T13:45:55.509 INFO:teuthology.orchestra.run.smithi069.stderr:ragweed.tests.tests.r_test_multipart_index_versioning.test ... ERROR 2023-09-22T13:45:56.522 INFO:teuthology.orchestra.run.smithi069.stderr:ragweed.tests.tests.r_test_multipart_simple.test ... ok 2023-09-22T13:46:01.779 INFO:teuthology.orchestra.run.smithi069.stderr:ragweed.tests.tests.r_test_obj_storage_class.test ... ok 2023-09-22T13:46:04.561 INFO:teuthology.orchestra.run.smithi069.stderr:ragweed.tests.tests.r_test_obj_storage_class_copy.test ... ok 2023-09-22T13:46:06.448 INFO:teuthology.orchestra.run.smithi069.stderr:ragweed.tests.tests.r_test_obj_storage_class_multipart.test ... ok 2023-09-22T13:46:10.400 INFO:teuthology.orchestra.run.smithi069.stderr:ragweed.tests.tests.r_test_obj_storage_class_multipart_copy.test ... ok 2023-09-22T13:46:13.133 INFO:teuthology.orchestra.run.smithi069.stderr:ragweed.tests.tests.r_test_small_obj_data.test ... ok 2023-09-22T13:46:13.133 INFO:teuthology.orchestra.run.smithi069.stderr: 2023-09-22T13:46:13.133 INFO:teuthology.orchestra.run.smithi069.stderr:====================================================================== 2023-09-22T13:46:13.133 INFO:teuthology.orchestra.run.smithi069.stderr:ERROR: ragweed.tests.tests.r_test_multipart_index_versioning.test 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr:---------------------------------------------------------------------- 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr:Traceback (most recent call last): 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr: self.test(*self.arg) 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/framework.py", line 345, in test 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr: self.check() 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/tests/tests.py", line 296, in check 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr: index_check_result = rgwa().check_bucket_index(rb.name) 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/framework.py", line 68, in check_bucket_index 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr: return self.get_resource('/admin/bucket',{'index' : None, 'bucket':bucket_name}) 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/framework.py", line 50, in get_resource 2023-09-22T13:46:13.134 INFO:teuthology.orchestra.run.smithi069.stderr: return munch.munchify(json.loads(r.read())) 2023-09-22T13:46:13.135 INFO:teuthology.orchestra.run.smithi069.stderr: File "/usr/lib64/python3.6/json/__init__.py", line 354, in loads 2023-09-22T13:46:13.135 INFO:teuthology.orchestra.run.smithi069.stderr: return _default_decoder.decode(s) 2023-09-22T13:46:13.135 INFO:teuthology.orchestra.run.smithi069.stderr: File "/usr/lib64/python3.6/json/decoder.py", line 342, in decode 2023-09-22T13:46:13.135 INFO:teuthology.orchestra.run.smithi069.stderr: raise JSONDecodeError("Extra data", s, end) 2023-09-22T13:46:13.135 INFO:teuthology.orchestra.run.smithi069.stderr:json.decoder.JSONDecodeError: Extra data: line 1 column 3 (char 2) 2023-09-22T13:46:13.135 INFO:teuthology.orchestra.run.smithi069.stderr:-------------------- >> begin captured stdout << --------------------- 2023-09-22T13:46:13.135 INFO:teuthology.orchestra.run.smithi069.stderr:Munch({'key': 'bucket:test-client.0-r-test-multipart-index-versioning.1', 'ver': Munch({'tag': '__BU3WpPyo7pdmNj3_VQkYCk', 'ver': 1}), 'mtime': '2023-09-22T13:45:54.550793Z', 'data': Munch({'bucket': Munch({'name': 'test-client.0-r-test-multipart-index-versioning.1', 'marker': 'f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.5', 'bucket_id': 'f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.5', 'tenant': '', 'explicit_placement': Munch({'data_pool': '', 'data_extra_pool': '', 'index_pool': ''})}), 'owner': 'ragweed.client.0', 'creation_time': '2023-09-22T13:45:54.543269Z', 'linked': 'true', 'has_bucket_info': 'false'})}) 2023-09-22T13:46:13.135 INFO:teuthology.orchestra.run.smithi069.stderr:bucket_info: Munch({'bucket': Munch({'name': 'test-client.0-r-test-multipart-index-versioning.1', 'marker': 'f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.5', 'bucket_id': 'f54d190c-30d0-4b87-88c4-5bcc7de96de4.4686.5', 'tenant': '', 'explicit_placement': Munch({'data_pool': '', 'data_extra_pool': '', 'index_pool': ''})}), 'creation_time': '2023-09-22T13:45:54.543269Z', 'owner': 'ragweed.client.0', 'flags': 0, 'zonegroup': 'f2fb3710-6bf4-40b4-b2d8-9afc2276602f', 'placement_rule': 'default-placement', 'has_instance_obj': 'false', 'quota': Munch({'enabled': False, 'check_on_raw': False, 'max_size': -1, 'max_size_kb': 0, 'max_objects': -1}), 'num_shards': 11, 'bi_shard_hash_type': 0, 'requester_pays': 'false', 'has_website': 'false', 'swift_versioning': 'false', 'swift_ver_location': '', 'index_type': 0, 'mdsearch_config': [], 'reshard_status': 0, 'new_bucket_instance_id': ''}) 2023-09-22T13:46:13.135 INFO:teuthology.orchestra.run.smithi069.stderr:read_test_data= b'{"r_buckets": [{"__pickle": "ccopy_reg\\n_reconstructor\\np0\\n(cragweed.framework\\nRBucket\\np1\\nc__builtin__\\nobject\\np2\\nNtp3\\nRp4\\n(dp5\\nVzone\\np6\\ng0\\n(cragweed.framework\\nRZone\\np7\\ng2\\nNtp8\\nRp9\\n(dp10\\nVconn\\np11\\ng0\\n(cmunch\\nMunch\\np12\\nc__builtin__\\ndict\\np13\\n(dp14\\nVsystem\\np15\\ng0\\n(cragweed.framework\\nRGWConnection\\np16\\ng2\\nNtp17\\nRp18\\n(dp19\\nVhost\\np20\\nVsmithi069.front.sepia.ceph.com\\np21\\nsVport\\np22\\nL443L\\nsVis_secure\\np23\\nI01\\nsg11\\ng0\\n(cboto.s3.connection\\nS3Connection\\np24\\ng2\\nNtp25\\nRp26\\n(dp27\\nVcalling_format\\np28\\ng0\\n(cboto.s3.connection\\nOrdinaryCallingFormat\\np29\\ng2\\nNtp30\\nRp31\\nsVbucket_class\\np32\\ncboto.s3.bucket\\nBucket\\np33\\nsVanon\\np34\\nI00\\nsVsuppress_consec_slashes\\np35\\nI01\\nsVnum_retries\\np36\\nL6L\\nsg23\\nI01\\nsVhttps_validate_certificates\\np37\\nNsVca_certificates_file\\np38\\nV/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/boto/cacerts/cacerts.txt\\np39\\nsg22\\nL443L\\nsVproxy\\np40\\nNsVproxy_port\\np41\\nNsVproxy_user\\np42\\nNsVproxy_pass\\np43\\nNsVno_proxy\\np44\\nV\\np45\\nsVuse_proxy\\np46\\nI00\\nsVhttp_exceptions\\np47\\n(chttplib\\nHTTPException\\np48\\ncexceptions\\nOSError\\np49\\ncsocket\\ngaierror\\np50\\nchttplib\\nBadStatusLine\\np51\\ntp52\\nsVhttp_unretryable_exceptions\\np53\\n(lp54\\ncboto.https_connection\\nInvalidCertificateException\\np55\\nasVsocket_exception_values\\np56\\n(L4L\\ntp57\\nsVhttps_connection_factory\\np58\\nNsVprotocol\\np59\\nVhttps\\np60\\nsg20\\ng21\\nsVpath\\np61\\nV/\\np62\\nsVdebug\\np63\\nL0L\\nsVhost_header\\np64\\nNsVhttp_connection_kwargs\\np65\\n(dp66\\nVtimeout\\np67\\nL30L\\nssV_provider_type\\np68\\nVaws\\np69\\nsVprovider\\np70\\ng0\\n(cboto.provider\\nProvider\\np71\\ng2\\nNtp72\\nRp73\\n(dp74\\ng20\\nNsg22\\nNsg64\\nNsV_access_key\\np75\\nVGRVTXEKBJSFREVTTDYYI\\np76\\nsV_secret_key\\np77\\nVJhuFkyWULldI4j74AAvGNc1Isbx6dJmGFiSxH5/TyUyRZwrLuBp6VA==\\np78\\nsV_security_token\\np79\\nNsVprofile_name\\np80\\nNsVname\\np81\\ng69\\nsVacl_class\\np82\\ncboto.s3.acl\\nPolicy\\np83\\nsVcanned_acls\\np84\\n(lp85\\nVprivate\\np86\\naVpublic-read\\np87\\naVpublic-read-write\\np88\\naVauthenticated-read\\np89\\naVbucket-owner-read\\np90\\naVbucket-owner-full-control\\np91\\naVlog-delivery-write\\np92\\nasV_credential_expiry_time\\np93\\nNsVshared_credentials\\np94\\ng0\\n(cboto.pyami.config\\nConfig\\np95\\ng2\\nNtp96\\nRp97\\n(dp98\\nV_parser\\np99\\ng0\\n(cConfigParser\\nConfigParser\\np100\\ng2\\nNtp101\\nRp102\\n(dp103\\nV_dict\\np104\\nccollections\\nOrderedDict\\np105\\nsV_sections\\np106\\ng105\\n(tRp107\\nsV_defaults\\np108\\ng105\\n(tRp109\\nVworking_dir\\np110\\nV/mnt/pyami\\np111\\nsVdebug\\np112\\nV0\\np113\\nssV_converters\\np114\\ng0\\n(cConfigParser\\nConverterMapping\\np115\\ng2\\nNtp116\\nRp117\\n(dp118\\ng99\\ng102\\nsV_data\\np119\\n(dp120\\nVboolean\\np121\\nNsVfloat\\np122\\nNsVint\\np123\\nNssbsV_proxies\\np124\\ng105\\n(tRp125\\nVDEFAULT\\np126\\ng0\\n(cConfigParser\\nSectionProxy\\np127\\ng2\\nNtp128\\nRp129\\n(dp130\\ng99\\ng102\\nsV_name\\np131\\ng126\\nsVgetboolean\\np132\\ncfunctools\\npartial\\np133\\n(c__builtin__\\ngetattr\\np134\\n(g129\\nVget\\np135\\ntp136\\nRp137\\ntp138\\nRp139\\n(g137\\n(t(dp140\\nV_impl\\np141\\ng134\\n(g102\\ng132\\ntp142\\nRp143\\nsNtp144\\nbsVgetfloat\\np145\\ng133\\n(g134\\n(g129\\ng135\\ntp146\\nRp147\\ntp148\\nRp149\\n(g147\\n(t(dp150\\ng141\\ng134\\n(g102\\ng145\\ntp151\\nRp152\\nsNtp153\\nbsVgetint\\np154\\ng133\\n(g134\\n(g129\\ng135\\ntp155\\nRp156\\ntp157\\nRp158\\n(g156\\n(t(dp159\\ng141\\ng134\\n(g102\\ng154\\ntp160\\nRp161\\nsNtp162\\nbsbssV_delimiters\\np163\\n(V=\\np164\\nV:\\np165\\ntp166\\nsV_optcre\\np167\\ncre\\n_compile\\np168\\n(V\\\\u000a (?P