2023-09-22T11:37:40.484 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-22T11:37:40.485 DEBUG:teuthology.run:Teuthology command: teuthology --description rgw/verify/{centos_latest clusters/fixed-2 frontend/civetweb ignore-pg-availability msgr-failures/few objectstore/bluestore-bitmap overrides proto/https rgw_pool_type/ec sharding$/{default} striping$/{stripe-greater-than-chunk} tasks/{0-install bucket-check cls ragweed reshard s3tests-java s3tests versioning} validater/valgrind} --verbose --name cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi --archive /home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400916 --owner scheduled_cfsnyder@teuthology -- /home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400916/orig.config.yaml 2023-09-22T11:37:40.526 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-22T11:37:40.599 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400916 branch: wip-cfsnyder-bucket-check-pacific description: rgw/verify/{centos_latest clusters/fixed-2 frontend/civetweb ignore-pg-availability msgr-failures/few objectstore/bluestore-bitmap overrides proto/https rgw_pool_type/ec sharding$/{default} striping$/{stripe-greater-than-chunk} tasks/{0-install bucket-check cls ragweed reshard s3tests-java s3tests versioning} validater/valgrind} email: null first_in_suite: false job_id: '7400916' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: wip-cfsnyder-bucket-check-pacific ceph: conf: client: debug rgw: 20 rgw crypt require ssl: false rgw crypt s3 kms backend: testing rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= rgw lc debug interval: 10 rgw max chunk size: 4194304 rgw obj stripe size: 6291456 setgroup: ceph setuser: ceph global: mon client directed command retry: 5 ms inject socket failures: 5000 osd heartbeat grace: 40 osd_max_pg_log_entries: 10 osd_min_pg_log_entries: 10 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon osd crush smoke test: false osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd fast shutdown: false osd objectstore: bluestore osd shutdown pgref assert: true flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(PG_AVAILABILITY\) - \(POOL_APP_NOT_ENABLED\) - \(OSD_SLOW_PING_TIME sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 openssl_keys: rgw.client.0: ca: root client: client.0 embed-key: true rgw.client.1: ca: root client: client.1 embed-key: true root: client: client.0 cn: teuthology install: - client.0 - client.1 key-type: rsa:4096 rgw: client.0: ssl certificate: rgw.client.0 client.1: ssl certificate: rgw.client.1 compression type: random ec-data-pool: true frontend: civetweb storage classes: LUKEWARM, FROZEN s3tests: slow_backend: true selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-cfsnyder-bucket-check-pacific sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 owner: scheduled_cfsnyder@teuthology priority: 101 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - node-exporter.b sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 sleep_before_teardown: 0 suite: rgw suite_branch: wip-cfsnyder-bucket-check-pacific suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 targets: smithi170.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGW1RWBfQqQYn2+WYnqrlGDOOPGyQm5QE4qsRK/5elTEgdVL6uLQVo5gdgU01NNAlnLIf5QJorVaKeIFPv6zhmg= smithi191.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDh6tNYcj7aus9EYSbirgEOwv1/CZc8dW4p2jSWN9n/V1HkaO8NCSJSbt2T3vIm134ruxzApHaWCliSSjzp224s= tasks: - install: null - ceph: null - openssl_keys: null - rgw: client.0: null - workunit: clients: client.0: - rgw/run-bucket-check.sh - workunit: clients: client.0: - cls/test_cls_lock.sh - cls/test_cls_log.sh - cls/test_cls_refcount.sh - cls/test_cls_rgw.sh - cls/test_cls_cmpomap.sh - cls/test_cls_2pc_queue.sh - rgw/test_rgw_gc_log.sh - rgw/test_rgw_obj.sh - rgw/test_rgw_throttle.sh - rgw/test_librgw_file.sh - ragweed: client.0: default-branch: ceph-pacific rgw_server: client.0 stages: prepare,check - workunit: clients: client.0: - rgw/run-reshard.sh - s3tests-java: client.0: force-branch: ceph-pacific force-repo: https://github.com/ceph/java_s3tests.git - s3tests: client.0: force-branch: ceph-pacific rgw_server: client.0 - workunit: clients: client.0: - rgw/run-versioning.sh teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-22_11:28:34 tube: smithi user: cfsnyder verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-22T11:37:40.600 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa; will attempt to use it 2023-09-22T11:37:40.600 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa/tasks 2023-09-22T11:37:40.600 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-22T11:37:40.602 INFO:teuthology.task.internal:Checking packages... 2023-09-22T11:37:40.624 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '31c126b7c1c0c06b66d9d19c5bfc51c46b503392' 2023-09-22T11:37:40.624 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-22T11:37:40.624 INFO:teuthology.packaging:ref: None 2023-09-22T11:37:40.624 INFO:teuthology.packaging:tag: None 2023-09-22T11:37:40.624 INFO:teuthology.packaging:branch: wip-cfsnyder-bucket-check-pacific 2023-09-22T11:37:40.624 INFO:teuthology.packaging:sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T11:37:40.624 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-cfsnyder-bucket-check-pacific 2023-09-22T11:37:40.848 INFO:teuthology.task.internal:Found packages for ceph version 16.2.14-5.g31c126b7 2023-09-22T11:37:40.848 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-22T11:37:40.858 INFO:teuthology.task.internal:no buildpackages task found 2023-09-22T11:37:40.858 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-22T11:37:40.909 INFO:teuthology.task.internal:Saving configuration 2023-09-22T11:37:40.923 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-22T11:37:40.951 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-22T11:37:40.978 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi170.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400916', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-22 11:29:45.032347', 'locked_by': 'scheduled_cfsnyder@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGW1RWBfQqQYn2+WYnqrlGDOOPGyQm5QE4qsRK/5elTEgdVL6uLQVo5gdgU01NNAlnLIf5QJorVaKeIFPv6zhmg='} 2023-09-22T11:37:40.999 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi191.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400916', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-22 11:29:45.030432', 'locked_by': 'scheduled_cfsnyder@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDh6tNYcj7aus9EYSbirgEOwv1/CZc8dW4p2jSWN9n/V1HkaO8NCSJSbt2T3vIm134ruxzApHaWCliSSjzp224s='} 2023-09-22T11:37:40.999 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-22T11:37:41.010 INFO:teuthology.task.internal:roles: ubuntu@smithi170.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a'] 2023-09-22T11:37:41.010 INFO:teuthology.task.internal:roles: ubuntu@smithi191.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'node-exporter.b'] 2023-09-22T11:37:41.010 INFO:teuthology.run_tasks:Running task console_log... 2023-09-22T11:37:41.112 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f77d6668ee0>, signals=[15]) 2023-09-22T11:37:41.113 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-22T11:37:41.124 INFO:teuthology.task.internal:Opening connections... 2023-09-22T11:37:41.124 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi170.front.sepia.ceph.com 2023-09-22T11:37:41.125 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi170.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T11:37:41.192 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi191.front.sepia.ceph.com 2023-09-22T11:37:41.193 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T11:37:41.259 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-22T11:37:41.265 DEBUG:teuthology.orchestra.run.smithi170:> uname -m 2023-09-22T11:37:41.288 INFO:teuthology.orchestra.run.smithi170.stdout:x86_64 2023-09-22T11:37:41.288 DEBUG:teuthology.orchestra.run.smithi170:> cat /etc/os-release 2023-09-22T11:37:41.349 INFO:teuthology.orchestra.run.smithi170.stdout:NAME="CentOS Stream" 2023-09-22T11:37:41.350 INFO:teuthology.orchestra.run.smithi170.stdout:VERSION="8" 2023-09-22T11:37:41.350 INFO:teuthology.orchestra.run.smithi170.stdout:ID="centos" 2023-09-22T11:37:41.350 INFO:teuthology.orchestra.run.smithi170.stdout:ID_LIKE="rhel fedora" 2023-09-22T11:37:41.350 INFO:teuthology.orchestra.run.smithi170.stdout:VERSION_ID="8" 2023-09-22T11:37:41.350 INFO:teuthology.orchestra.run.smithi170.stdout:PLATFORM_ID="platform:el8" 2023-09-22T11:37:41.350 INFO:teuthology.orchestra.run.smithi170.stdout:PRETTY_NAME="CentOS Stream 8" 2023-09-22T11:37:41.350 INFO:teuthology.orchestra.run.smithi170.stdout:ANSI_COLOR="0;31" 2023-09-22T11:37:41.350 INFO:teuthology.orchestra.run.smithi170.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-09-22T11:37:41.351 INFO:teuthology.orchestra.run.smithi170.stdout:HOME_URL="https://centos.org/" 2023-09-22T11:37:41.351 INFO:teuthology.orchestra.run.smithi170.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-22T11:37:41.351 INFO:teuthology.orchestra.run.smithi170.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-22T11:37:41.351 INFO:teuthology.orchestra.run.smithi170.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-09-22T11:37:41.351 INFO:teuthology.lock.ops:Updating smithi170.front.sepia.ceph.com on lock server 2023-09-22T11:37:41.375 DEBUG:teuthology.orchestra.run.smithi191:> uname -m 2023-09-22T11:37:41.397 INFO:teuthology.orchestra.run.smithi191.stdout:x86_64 2023-09-22T11:37:41.397 DEBUG:teuthology.orchestra.run.smithi191:> cat /etc/os-release 2023-09-22T11:37:41.457 INFO:teuthology.orchestra.run.smithi191.stdout:NAME="CentOS Stream" 2023-09-22T11:37:41.457 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION="8" 2023-09-22T11:37:41.458 INFO:teuthology.orchestra.run.smithi191.stdout:ID="centos" 2023-09-22T11:37:41.458 INFO:teuthology.orchestra.run.smithi191.stdout:ID_LIKE="rhel fedora" 2023-09-22T11:37:41.458 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION_ID="8" 2023-09-22T11:37:41.458 INFO:teuthology.orchestra.run.smithi191.stdout:PLATFORM_ID="platform:el8" 2023-09-22T11:37:41.458 INFO:teuthology.orchestra.run.smithi191.stdout:PRETTY_NAME="CentOS Stream 8" 2023-09-22T11:37:41.458 INFO:teuthology.orchestra.run.smithi191.stdout:ANSI_COLOR="0;31" 2023-09-22T11:37:41.458 INFO:teuthology.orchestra.run.smithi191.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-09-22T11:37:41.458 INFO:teuthology.orchestra.run.smithi191.stdout:HOME_URL="https://centos.org/" 2023-09-22T11:37:41.458 INFO:teuthology.orchestra.run.smithi191.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-22T11:37:41.458 INFO:teuthology.orchestra.run.smithi191.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-22T11:37:41.458 INFO:teuthology.orchestra.run.smithi191.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-09-22T11:37:41.459 INFO:teuthology.lock.ops:Updating smithi191.front.sepia.ceph.com on lock server 2023-09-22T11:37:41.482 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-22T11:37:41.493 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-22T11:37:41.555 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-22T11:37:41.555 DEBUG:teuthology.orchestra.run.smithi170:> test '!' -e /home/ubuntu/cephtest 2023-09-22T11:37:41.558 DEBUG:teuthology.orchestra.run.smithi191:> test '!' -e /home/ubuntu/cephtest 2023-09-22T11:37:41.578 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-22T11:37:41.589 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-22T11:37:41.589 DEBUG:teuthology.orchestra.run.smithi170:> test -z $(ls -A /var/lib/ceph) 2023-09-22T11:37:41.618 DEBUG:teuthology.orchestra.run.smithi191:> test -z $(ls -A /var/lib/ceph) 2023-09-22T11:37:41.655 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-22T11:37:41.718 INFO:teuthology.run_tasks:Running task kernel... 2023-09-22T11:37:41.737 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-22T11:37:41.737 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-22T11:37:41.738 DEBUG:teuthology.orchestra.run.smithi170:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-22T11:37:41.738 DEBUG:teuthology.orchestra.run.smithi191:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-22T11:37:41.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T11:37:41.759 DEBUG:teuthology.orchestra.run.smithi170:> uname -r 2023-09-22T11:37:41.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T11:37:41.760 DEBUG:teuthology.orchestra.run.smithi191:> uname -r 2023-09-22T11:37:41.820 INFO:teuthology.orchestra.run.smithi170.stdout:4.18.0-489.el8.x86_64 2023-09-22T11:37:41.821 INFO:teuthology.task.kernel:Running kernel on smithi170: 4.18.0-489.el8.x86_64 2023-09-22T11:37:41.821 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum install -y kernel 2023-09-22T11:37:41.822 INFO:teuthology.orchestra.run.smithi191.stdout:4.18.0-489.el8.x86_64 2023-09-22T11:37:41.822 INFO:teuthology.task.kernel:Running kernel on smithi191: 4.18.0-489.el8.x86_64 2023-09-22T11:37:41.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum install -y kernel 2023-09-22T11:58:58.774 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 8 - AppStream 27 kB/s | 33 MB 21:14 2023-09-22T11:59:03.093 INFO:teuthology.orchestra.run.smithi170.stdout:CentOS Stream 8 - AppStream 27 kB/s | 33 MB 21:18 2023-09-22T11:59:06.748 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 8 - BaseOS 25 MB/s | 47 MB 00:01 2023-09-22T11:59:11.159 INFO:teuthology.orchestra.run.smithi170.stdout:CentOS Stream 8 - BaseOS 23 MB/s | 47 MB 00:02 2023-09-22T11:59:13.840 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 8 - Extras 50 kB/s | 18 kB 00:00 2023-09-22T11:59:14.683 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 8 - Extras common packages 10 kB/s | 6.9 kB 00:00 2023-09-22T11:59:14.993 INFO:teuthology.orchestra.run.smithi191.stdout:Copr repo for python3-asyncssh owned by ceph 22 kB/s | 3.5 kB 00:00 2023-09-22T11:59:16.694 INFO:teuthology.orchestra.run.smithi191.stdout:Extra Packages for Enterprise Linux 10 MB/s | 16 MB 00:01 2023-09-22T11:59:18.350 INFO:teuthology.orchestra.run.smithi170.stdout:CentOS Stream 8 - Extras 48 kB/s | 18 kB 00:00 2023-09-22T11:59:18.851 INFO:teuthology.orchestra.run.smithi170.stdout:CentOS Stream 8 - Extras common packages 21 kB/s | 6.9 kB 00:00 2023-09-22T11:59:19.152 INFO:teuthology.orchestra.run.smithi170.stdout:Copr repo for python3-asyncssh owned by ceph 23 kB/s | 3.5 kB 00:00 2023-09-22T11:59:20.503 INFO:teuthology.orchestra.run.smithi191.stdout:lab-extras 419 kB/s | 24 kB 00:00 2023-09-22T11:59:24.085 INFO:teuthology.orchestra.run.smithi191.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-09-22T11:59:24.085 INFO:teuthology.orchestra.run.smithi191.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-09-22T11:59:24.327 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T11:59:24.328 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T11:59:24.328 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2023-09-22T11:59:24.328 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T11:59:24.328 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T11:59:24.328 INFO:teuthology.orchestra.run.smithi191.stdout: kernel x86_64 4.18.0-513.el8 baseos 10 M 2023-09-22T11:59:24.328 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-09-22T11:59:24.328 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-core x86_64 4.18.0-513.el8 baseos 43 M 2023-09-22T11:59:24.328 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-modules x86_64 4.18.0-513.el8 baseos 35 M 2023-09-22T11:59:24.329 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T11:59:24.329 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T11:59:24.329 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T11:59:24.329 INFO:teuthology.orchestra.run.smithi191.stdout:Install 3 Packages 2023-09-22T11:59:24.329 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T11:59:24.329 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 88 M 2023-09-22T11:59:24.329 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 96 M 2023-09-22T11:59:24.330 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T11:59:25.762 INFO:teuthology.orchestra.run.smithi191.stdout:(1/3): kernel-4.18.0-513.el8.x86_64.rpm 8.1 MB/s | 10 MB 00:01 2023-09-22T11:59:27.133 INFO:teuthology.orchestra.run.smithi191.stdout:(2/3): kernel-core-4.18.0-513.el8.x86_64.rpm 16 MB/s | 43 MB 00:02 2023-09-22T11:59:32.221 INFO:teuthology.orchestra.run.smithi191.stdout:(3/3): kernel-modules-4.18.0-513.el8.x86_64.rpm 4.6 MB/s | 35 MB 00:07 2023-09-22T11:59:32.221 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T11:59:32.221 INFO:teuthology.orchestra.run.smithi191.stdout:Total 11 MB/s | 88 MB 00:07 2023-09-22T11:59:32.894 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T11:59:33.326 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T11:59:33.326 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T11:59:34.557 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T11:59:34.577 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T11:59:37.329 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T11:59:37.835 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T11:59:41.433 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T11:59:41.867 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T11:59:45.238 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T11:59:45.523 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:00:07.334 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:00:07.359 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:00:07.700 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:00:07.700 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : kernel-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:00:07.700 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : kernel-core-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:00:08.456 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:00:08.456 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:00:08.456 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T12:00:08.456 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-4.18.0-513.el8.x86_64 kernel-core-4.18.0-513.el8.x86_64 2023-09-22T12:00:08.457 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-modules-4.18.0-513.el8.x86_64 2023-09-22T12:00:08.457 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:00:08.457 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T12:00:08.594 DEBUG:teuthology.orchestra.run.smithi191:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T12:00:08.814 INFO:teuthology.orchestra.run.smithi191.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T12:00:08.814 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T12:00:08.814 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-513.el8.x86_64 2023-09-22T12:00:08.814 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-09-22T12:00:08.814 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-09-22T12:00:08.814 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-09-22T12:00:08.815 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi191.front.sepia.ceph.com, path=None, version=distro) 2023-09-22T12:00:08.815 DEBUG:teuthology.orchestra.run.smithi191:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T12:00:09.024 INFO:teuthology.orchestra.run.smithi191.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T12:00:09.024 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T12:00:09.025 DEBUG:teuthology.orchestra.run.smithi191:> sudo rpm -qi grub2-tools 2023-09-22T12:00:09.058 INFO:teuthology.orchestra.run.smithi191.stdout:Name : grub2-tools 2023-09-22T12:00:09.058 INFO:teuthology.orchestra.run.smithi191.stdout:Epoch : 1 2023-09-22T12:00:09.058 INFO:teuthology.orchestra.run.smithi191.stdout:Version : 2.02 2023-09-22T12:00:09.058 INFO:teuthology.orchestra.run.smithi191.stdout:Release : 129.el8 2023-09-22T12:00:09.058 INFO:teuthology.orchestra.run.smithi191.stdout:Architecture: x86_64 2023-09-22T12:00:09.058 INFO:teuthology.orchestra.run.smithi191.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:Group : System Environment/Base 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:Size : 9265586 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:License : GPLv3+ 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:Build Host : kbuilder.bsys.centos.org 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:Relocations : (not relocatable) 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:Packager : CentOS BuildSystem 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:Vendor : CentOS 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:URL : http://www.gnu.org/software/grub/ 2023-09-22T12:00:09.059 INFO:teuthology.orchestra.run.smithi191.stdout:Summary : Support tools for GRUB. 2023-09-22T12:00:09.060 INFO:teuthology.orchestra.run.smithi191.stdout:Description : 2023-09-22T12:00:09.060 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:00:09.060 INFO:teuthology.orchestra.run.smithi191.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-09-22T12:00:09.060 INFO:teuthology.orchestra.run.smithi191.stdout:customizable bootloader with modular architecture. It supports a rich 2023-09-22T12:00:09.060 INFO:teuthology.orchestra.run.smithi191.stdout:variety of kernel formats, file systems, computer architectures and 2023-09-22T12:00:09.060 INFO:teuthology.orchestra.run.smithi191.stdout:hardware devices. 2023-09-22T12:00:09.060 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:00:09.060 INFO:teuthology.orchestra.run.smithi191.stdout:This subpackage provides tools for support of all platforms. 2023-09-22T12:00:09.061 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-09-22T12:00:09.061 INFO:teuthology.task.kernel:Updating grub on smithi191 to boot 4.18.0-513.el8.x86_64 2023-09-22T12:00:09.061 DEBUG:teuthology.orchestra.run.smithi191:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-09-22T12:00:10.698 INFO:teuthology.orchestra.run.smithi191.stderr:Generating grub configuration file ... 2023-09-22T12:00:13.170 INFO:teuthology.orchestra.run.smithi191.stderr:done 2023-09-22T12:00:13.171 DEBUG:teuthology.orchestra.run.smithi191:> mktemp 2023-09-22T12:00:13.194 INFO:teuthology.orchestra.run.smithi191.stdout:/tmp/tmp.8MeVuDKf99 2023-09-22T12:00:13.195 DEBUG:teuthology.orchestra.run.smithi191:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.8MeVuDKf99 2023-09-22T12:00:13.269 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod 0666 /tmp/tmp.8MeVuDKf99 2023-09-22T12:00:13.435 DEBUG:teuthology.orchestra.remote:smithi191:/tmp/tmp.8MeVuDKf99 is 6KB 2023-09-22T12:00:13.483 DEBUG:teuthology.orchestra.run.smithi191:> rm -fr /tmp/tmp.8MeVuDKf99 2023-09-22T12:00:13.501 DEBUG:teuthology.orchestra.run.smithi191:> sudo /bin/ls /boot/loader/entries || true 2023-09-22T12:00:13.573 INFO:teuthology.orchestra.run.smithi191.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-09-22T12:00:13.573 INFO:teuthology.orchestra.run.smithi191.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-09-22T12:00:13.574 INFO:teuthology.orchestra.run.smithi191.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-09-22T12:00:13.574 INFO:teuthology.orchestra.run.smithi191.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64.conf 2023-09-22T12:00:13.575 DEBUG:teuthology.orchestra.run.smithi191:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64 2023-09-22T12:00:13.734 DEBUG:teuthology.orchestra.run.smithi191:> sudo shutdown -r now 2023-09-22T12:00:34.012 INFO:teuthology.orchestra.run.smithi170.stdout:Extra Packages for Enterprise Linux 217 kB/s | 16 MB 01:14 2023-09-22T12:00:37.802 INFO:teuthology.orchestra.run.smithi170.stdout:lab-extras 288 kB/s | 24 kB 00:00 2023-09-22T12:00:41.389 INFO:teuthology.orchestra.run.smithi170.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-09-22T12:00:41.390 INFO:teuthology.orchestra.run.smithi170.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-09-22T12:00:41.633 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:00:41.634 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:00:41.634 INFO:teuthology.orchestra.run.smithi170.stdout: Package Architecture Version Repository Size 2023-09-22T12:00:41.634 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:00:41.634 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T12:00:41.635 INFO:teuthology.orchestra.run.smithi170.stdout: kernel x86_64 4.18.0-513.el8 baseos 10 M 2023-09-22T12:00:41.635 INFO:teuthology.orchestra.run.smithi170.stdout:Installing dependencies: 2023-09-22T12:00:41.635 INFO:teuthology.orchestra.run.smithi170.stdout: kernel-core x86_64 4.18.0-513.el8 baseos 43 M 2023-09-22T12:00:41.635 INFO:teuthology.orchestra.run.smithi170.stdout: kernel-modules x86_64 4.18.0-513.el8 baseos 35 M 2023-09-22T12:00:41.635 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:00:41.635 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T12:00:41.635 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:00:41.635 INFO:teuthology.orchestra.run.smithi170.stdout:Install 3 Packages 2023-09-22T12:00:41.635 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:00:41.635 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 88 M 2023-09-22T12:00:41.636 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 96 M 2023-09-22T12:00:41.636 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T12:00:42.794 INFO:teuthology.orchestra.run.smithi170.stdout:(1/3): kernel-4.18.0-513.el8.x86_64.rpm 10 MB/s | 10 MB 00:00 2023-09-22T12:00:43.737 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-09-22T12:00:43.737 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2023-09-22T12:00:43.738 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:00:43.761 INFO:teuthology.orchestra.run.smithi170.stdout:(2/3): kernel-modules-4.18.0-513.el8.x86_64.rpm 18 MB/s | 35 MB 00:01 2023-09-22T12:00:44.353 INFO:teuthology.orchestra.run.smithi170.stdout:(3/3): kernel-core-4.18.0-513.el8.x86_64.rpm 17 MB/s | 43 MB 00:02 2023-09-22T12:00:44.354 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:00:44.354 INFO:teuthology.orchestra.run.smithi170.stdout:Total 32 MB/s | 88 MB 00:02 2023-09-22T12:00:45.023 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T12:00:45.444 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T12:00:45.444 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T12:00:46.663 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T12:00:46.693 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T12:00:49.492 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T12:00:49.976 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:00:53.604 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:00:54.025 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:00:57.396 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:00:57.712 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:01:18.202 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.191 2023-09-22T12:01:19.439 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:01:19.462 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:01:19.804 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:01:19.804 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : kernel-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:01:19.804 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : kernel-core-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:01:20.498 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:01:20.499 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:01:20.499 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T12:01:20.499 INFO:teuthology.orchestra.run.smithi170.stdout: kernel-4.18.0-513.el8.x86_64 kernel-core-4.18.0-513.el8.x86_64 2023-09-22T12:01:20.499 INFO:teuthology.orchestra.run.smithi170.stdout: kernel-modules-4.18.0-513.el8.x86_64 2023-09-22T12:01:20.499 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:01:20.499 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T12:01:20.689 DEBUG:teuthology.orchestra.run.smithi170:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T12:01:20.910 INFO:teuthology.orchestra.run.smithi170.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T12:01:20.910 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T12:01:20.910 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-513.el8.x86_64 2023-09-22T12:01:20.910 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-09-22T12:01:20.910 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-09-22T12:01:20.911 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-09-22T12:01:20.911 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi170.front.sepia.ceph.com, path=None, version=distro) 2023-09-22T12:01:20.911 DEBUG:teuthology.orchestra.run.smithi170:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T12:01:21.119 INFO:teuthology.orchestra.run.smithi170.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T12:01:21.119 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T12:01:21.119 DEBUG:teuthology.orchestra.run.smithi170:> sudo rpm -qi grub2-tools 2023-09-22T12:01:21.152 INFO:teuthology.orchestra.run.smithi170.stdout:Name : grub2-tools 2023-09-22T12:01:21.152 INFO:teuthology.orchestra.run.smithi170.stdout:Epoch : 1 2023-09-22T12:01:21.153 INFO:teuthology.orchestra.run.smithi170.stdout:Version : 2.02 2023-09-22T12:01:21.153 INFO:teuthology.orchestra.run.smithi170.stdout:Release : 129.el8 2023-09-22T12:01:21.153 INFO:teuthology.orchestra.run.smithi170.stdout:Architecture: x86_64 2023-09-22T12:01:21.153 INFO:teuthology.orchestra.run.smithi170.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-09-22T12:01:21.153 INFO:teuthology.orchestra.run.smithi170.stdout:Group : System Environment/Base 2023-09-22T12:01:21.153 INFO:teuthology.orchestra.run.smithi170.stdout:Size : 9265586 2023-09-22T12:01:21.153 INFO:teuthology.orchestra.run.smithi170.stdout:License : GPLv3+ 2023-09-22T12:01:21.153 INFO:teuthology.orchestra.run.smithi170.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-09-22T12:01:21.153 INFO:teuthology.orchestra.run.smithi170.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-09-22T12:01:21.153 INFO:teuthology.orchestra.run.smithi170.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-09-22T12:01:21.153 INFO:teuthology.orchestra.run.smithi170.stdout:Build Host : kbuilder.bsys.centos.org 2023-09-22T12:01:21.154 INFO:teuthology.orchestra.run.smithi170.stdout:Relocations : (not relocatable) 2023-09-22T12:01:21.154 INFO:teuthology.orchestra.run.smithi170.stdout:Packager : CentOS BuildSystem 2023-09-22T12:01:21.154 INFO:teuthology.orchestra.run.smithi170.stdout:Vendor : CentOS 2023-09-22T12:01:21.154 INFO:teuthology.orchestra.run.smithi170.stdout:URL : http://www.gnu.org/software/grub/ 2023-09-22T12:01:21.154 INFO:teuthology.orchestra.run.smithi170.stdout:Summary : Support tools for GRUB. 2023-09-22T12:01:21.154 INFO:teuthology.orchestra.run.smithi170.stdout:Description : 2023-09-22T12:01:21.154 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:01:21.154 INFO:teuthology.orchestra.run.smithi170.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-09-22T12:01:21.154 INFO:teuthology.orchestra.run.smithi170.stdout:customizable bootloader with modular architecture. It supports a rich 2023-09-22T12:01:21.154 INFO:teuthology.orchestra.run.smithi170.stdout:variety of kernel formats, file systems, computer architectures and 2023-09-22T12:01:21.154 INFO:teuthology.orchestra.run.smithi170.stdout:hardware devices. 2023-09-22T12:01:21.155 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:01:21.155 INFO:teuthology.orchestra.run.smithi170.stdout:This subpackage provides tools for support of all platforms. 2023-09-22T12:01:21.156 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-09-22T12:01:21.156 INFO:teuthology.task.kernel:Updating grub on smithi170 to boot 4.18.0-513.el8.x86_64 2023-09-22T12:01:21.156 DEBUG:teuthology.orchestra.run.smithi170:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-09-22T12:01:22.924 INFO:teuthology.orchestra.run.smithi170.stderr:Generating grub configuration file ... 2023-09-22T12:01:24.726 INFO:teuthology.orchestra.run.smithi170.stderr:done 2023-09-22T12:01:24.728 DEBUG:teuthology.orchestra.run.smithi170:> mktemp 2023-09-22T12:01:24.761 INFO:teuthology.orchestra.run.smithi170.stdout:/tmp/tmp.T7TlVaSAOg 2023-09-22T12:01:24.761 DEBUG:teuthology.orchestra.run.smithi170:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.T7TlVaSAOg 2023-09-22T12:01:24.835 DEBUG:teuthology.orchestra.run.smithi170:> sudo chmod 0666 /tmp/tmp.T7TlVaSAOg 2023-09-22T12:01:24.991 DEBUG:teuthology.orchestra.remote:smithi170:/tmp/tmp.T7TlVaSAOg is 6KB 2023-09-22T12:01:25.039 DEBUG:teuthology.orchestra.run.smithi170:> rm -fr /tmp/tmp.T7TlVaSAOg 2023-09-22T12:01:25.057 DEBUG:teuthology.orchestra.run.smithi170:> sudo /bin/ls /boot/loader/entries || true 2023-09-22T12:01:25.140 INFO:teuthology.orchestra.run.smithi170.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-09-22T12:01:25.140 INFO:teuthology.orchestra.run.smithi170.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-09-22T12:01:25.140 INFO:teuthology.orchestra.run.smithi170.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-09-22T12:01:25.140 INFO:teuthology.orchestra.run.smithi170.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64.conf 2023-09-22T12:01:25.141 DEBUG:teuthology.orchestra.run.smithi170:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64 2023-09-22T12:01:25.259 DEBUG:teuthology.orchestra.run.smithi170:> sudo shutdown -r now 2023-09-22T12:01:27.205 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2023-09-22T12:01:27.206 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:01:45.598 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.191 2023-09-22T12:01:55.270 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-09-22T12:01:55.270 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi170.front.sepia.ceph.com' 2023-09-22T12:01:55.271 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi170.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:01:57.602 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2023-09-22T12:01:57.602 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:01:57.905 DEBUG:teuthology.orchestra.run.smithi191:> true 2023-09-22T12:01:58.201 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi191.front.sepia.ceph.com' 2023-09-22T12:01:58.201 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-513.el8.x86_64"... 2023-09-22T12:01:58.202 DEBUG:teuthology.orchestra.run.smithi191:> uname -r 2023-09-22T12:01:58.218 INFO:teuthology.orchestra.run.smithi191.stdout:4.18.0-513.el8.x86_64 2023-09-22T12:01:58.219 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-513.el8.x86_64 vs 4.18.0-513.el8.x86_64 2023-09-22T12:01:58.219 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-09-22T12:01:58.219 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-09-22T12:01:59.220 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-22T12:01:59.220 DEBUG:teuthology.orchestra.run.smithi191:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-22T12:01:59.310 INFO:teuthology.orchestra.run.smithi191.stdout:ttyS1 2023-09-22T12:01:59.322 DEBUG:teuthology.parallel:result is None 2023-09-22T12:02:13.758 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.170 2023-09-22T12:02:22.759 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi170.front.sepia.ceph.com' 2023-09-22T12:02:22.760 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi170.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:03:22.762 DEBUG:teuthology.orchestra.remote:timed out 2023-09-22T12:03:34.763 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi170.front.sepia.ceph.com' 2023-09-22T12:03:34.764 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi170.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:03:35.107 DEBUG:teuthology.orchestra.run.smithi170:> true 2023-09-22T12:03:35.462 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi170.front.sepia.ceph.com' 2023-09-22T12:03:35.462 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-513.el8.x86_64"... 2023-09-22T12:03:35.463 DEBUG:teuthology.orchestra.run.smithi170:> uname -r 2023-09-22T12:03:35.481 INFO:teuthology.orchestra.run.smithi170.stdout:4.18.0-513.el8.x86_64 2023-09-22T12:03:35.481 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-513.el8.x86_64 vs 4.18.0-513.el8.x86_64 2023-09-22T12:03:35.481 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-09-22T12:03:35.481 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-09-22T12:03:36.482 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-22T12:03:36.482 DEBUG:teuthology.orchestra.run.smithi170:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-22T12:03:36.567 INFO:teuthology.orchestra.run.smithi170.stdout:ttyS1 2023-09-22T12:03:36.586 DEBUG:teuthology.parallel:result is None 2023-09-22T12:03:36.587 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-22T12:03:36.596 INFO:teuthology.task.internal:Creating test directory... 2023-09-22T12:03:36.597 DEBUG:teuthology.orchestra.run.smithi170:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-22T12:03:36.599 DEBUG:teuthology.orchestra.run.smithi191:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-22T12:03:36.623 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-22T12:03:36.642 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-22T12:03:36.693 INFO:teuthology.task.internal:Creating archive directory... 2023-09-22T12:03:36.693 DEBUG:teuthology.orchestra.run.smithi170:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-22T12:03:36.695 DEBUG:teuthology.orchestra.run.smithi191:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-22T12:03:36.827 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-22T12:03:36.834 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-22T12:03:36.834 DEBUG:teuthology.orchestra.run.smithi170:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-22T12:03:36.837 DEBUG:teuthology.orchestra.run.smithi191:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-22T12:03:36.875 INFO:teuthology.orchestra.run.smithi170.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:03:36.886 INFO:teuthology.orchestra.run.smithi170.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:03:36.899 INFO:teuthology.orchestra.run.smithi191.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:03:36.911 INFO:teuthology.orchestra.run.smithi191.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:03:36.913 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-22T12:03:36.920 INFO:teuthology.task.internal:Configuring sudo... 2023-09-22T12:03:36.920 DEBUG:teuthology.orchestra.run.smithi170:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-22T12:03:36.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-22T12:03:36.960 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-22T12:03:36.969 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-22T12:03:36.969 DEBUG:teuthology.orchestra.run.smithi170:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-22T12:03:37.000 DEBUG:teuthology.orchestra.run.smithi191:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-22T12:03:37.019 DEBUG:teuthology.orchestra.run.smithi170:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:03:37.104 DEBUG:teuthology.orchestra.run.smithi170:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:03:37.155 DEBUG:teuthology.orchestra.run.smithi170:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:03:37.242 DEBUG:teuthology.orchestra.run.smithi170:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:03:37.285 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T12:03:37.285 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-22T12:03:37.352 DEBUG:teuthology.orchestra.run.smithi191:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:03:37.402 DEBUG:teuthology.orchestra.run.smithi191:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:03:37.461 DEBUG:teuthology.orchestra.run.smithi191:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:03:37.508 DEBUG:teuthology.orchestra.run.smithi191:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:03:37.552 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T12:03:37.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-22T12:03:37.621 DEBUG:teuthology.orchestra.run.smithi170:> sudo service rsyslog restart 2023-09-22T12:03:37.623 DEBUG:teuthology.orchestra.run.smithi191:> sudo service rsyslog restart 2023-09-22T12:03:37.677 INFO:teuthology.orchestra.run.smithi170.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-22T12:03:37.725 INFO:teuthology.orchestra.run.smithi191.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-22T12:03:38.211 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-22T12:03:38.219 INFO:teuthology.task.internal:Starting timer... 2023-09-22T12:03:38.219 INFO:teuthology.run_tasks:Running task pcp... 2023-09-22T12:03:38.271 INFO:teuthology.run_tasks:Running task selinux... 2023-09-22T12:03:38.320 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-22T12:03:38.320 DEBUG:teuthology.orchestra.run.smithi170:> sudo service auditd rotate 2023-09-22T12:03:38.407 INFO:teuthology.orchestra.run.smithi170.stdout:Rotating logs: 2023-09-22T12:03:38.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo service auditd rotate 2023-09-22T12:03:38.495 INFO:teuthology.orchestra.run.smithi191.stdout:Rotating logs: 2023-09-22T12:03:38.497 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-22T12:03:38.497 DEBUG:teuthology.orchestra.run.smithi170:> /usr/sbin/getenforce 2023-09-22T12:03:38.569 INFO:teuthology.orchestra.run.smithi170.stdout:Permissive 2023-09-22T12:03:38.570 DEBUG:teuthology.orchestra.run.smithi191:> /usr/sbin/getenforce 2023-09-22T12:03:38.614 INFO:teuthology.orchestra.run.smithi191.stdout:Permissive 2023-09-22T12:03:38.615 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi170.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi191.front.sepia.ceph.com': 'permissive'} 2023-09-22T12:03:38.615 DEBUG:teuthology.orchestra.run.smithi170:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-22T12:03:38.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T12:03:38.644 DEBUG:teuthology.orchestra.run.smithi191:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-22T12:03:38.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T12:03:38.673 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-22T12:03:38.673 DEBUG:teuthology.orchestra.run.smithi170:> sudo /usr/sbin/setenforce permissive 2023-09-22T12:03:38.711 DEBUG:teuthology.orchestra.run.smithi191:> sudo /usr/sbin/setenforce permissive 2023-09-22T12:03:38.740 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-22T12:03:38.750 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-22T12:03:38.754 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-22T12:03:38.882 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-22T12:03:38.902 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-22T12:03:38.903 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi170.front.sepia.ceph.com,smithi191.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-22T12:22:37.743 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi170.front.sepia.ceph.com'), Remote(name='ubuntu@smithi191.front.sepia.ceph.com')] 2023-09-22T12:22:37.744 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi170.front.sepia.ceph.com' 2023-09-22T12:22:37.745 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi170.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:22:37.813 DEBUG:teuthology.orchestra.run.smithi170:> true 2023-09-22T12:22:37.880 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi170.front.sepia.ceph.com' 2023-09-22T12:22:37.881 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2023-09-22T12:22:37.881 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:22:37.949 DEBUG:teuthology.orchestra.run.smithi191:> true 2023-09-22T12:22:38.026 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi191.front.sepia.ceph.com' 2023-09-22T12:22:38.026 INFO:teuthology.run_tasks:Running task clock... 2023-09-22T12:22:38.036 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-22T12:22:38.037 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-22T12:22:38.037 DEBUG:teuthology.orchestra.run.smithi170:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-22T12:22:38.039 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-22T12:22:38.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-22T12:22:38.069 INFO:teuthology.orchestra.run.smithi170.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-22T12:22:38.083 INFO:teuthology.orchestra.run.smithi170.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-22T12:22:38.098 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-22T12:22:38.112 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-22T12:22:38.143 INFO:teuthology.orchestra.run.smithi170.stderr:sudo: ntpd: command not found 2023-09-22T12:22:38.155 INFO:teuthology.orchestra.run.smithi170.stdout:506 Cannot talk to daemon 2023-09-22T12:22:38.169 INFO:teuthology.orchestra.run.smithi170.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-22T12:22:38.178 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: ntpd: command not found 2023-09-22T12:22:38.182 INFO:teuthology.orchestra.run.smithi170.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-22T12:22:38.190 INFO:teuthology.orchestra.run.smithi191.stdout:506 Cannot talk to daemon 2023-09-22T12:22:38.204 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-22T12:22:38.217 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-22T12:22:38.248 INFO:teuthology.orchestra.run.smithi170.stderr:bash: ntpq: command not found 2023-09-22T12:22:38.251 INFO:teuthology.orchestra.run.smithi170.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-22T12:22:38.251 INFO:teuthology.orchestra.run.smithi170.stdout:=============================================================================== 2023-09-22T12:22:38.252 INFO:teuthology.orchestra.run.smithi170.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:22:38.252 INFO:teuthology.orchestra.run.smithi170.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:22:38.252 INFO:teuthology.orchestra.run.smithi170.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:22:38.252 INFO:teuthology.orchestra.run.smithi170.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:22:38.291 INFO:teuthology.orchestra.run.smithi191.stderr:bash: ntpq: command not found 2023-09-22T12:22:38.295 INFO:teuthology.orchestra.run.smithi191.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-22T12:22:38.295 INFO:teuthology.orchestra.run.smithi191.stdout:=============================================================================== 2023-09-22T12:22:38.296 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:22:38.296 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:22:38.296 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:22:38.296 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:22:38.296 INFO:teuthology.run_tasks:Running task install... 2023-09-22T12:22:38.305 DEBUG:teuthology.task.install:project ceph 2023-09-22T12:22:38.305 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}} 2023-09-22T12:22:38.305 DEBUG:teuthology.task.install:config {'flavor': 'default', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'} 2023-09-22T12:22:38.305 INFO:teuthology.task.install:Using flavor: default 2023-09-22T12:22:38.311 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-22T12:22:38.311 INFO:teuthology.task.install:extra packages: [] 2023-09-22T12:22:38.311 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'tag': None, 'wait_for_package': False} 2023-09-22T12:22:38.312 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T12:22:38.313 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'tag': None, 'wait_for_package': False} 2023-09-22T12:22:38.314 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T12:22:38.486 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/ 2023-09-22T12:22:38.486 INFO:teuthology.task.install.rpm:Package version is 16.2.14-5.g31c126b7 2023-09-22T12:22:38.581 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/ 2023-09-22T12:22:38.581 INFO:teuthology.task.install.rpm:Package version is 16.2.14-5.g31c126b7 2023-09-22T12:22:38.599 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-22T12:22:38.599 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T12:22:38.599 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-22T12:22:38.633 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-22T12:22:38.634 DEBUG:teuthology.orchestra.run.smithi191:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-22T12:22:38.701 DEBUG:teuthology.orchestra.run.smithi191:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-22T12:22:38.765 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-22T12:22:38.766 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T12:22:38.766 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-22T12:22:38.778 DEBUG:teuthology.orchestra.run.smithi191:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-22T12:22:38.795 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-22T12:22:38.795 DEBUG:teuthology.orchestra.run.smithi170:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-22T12:22:38.845 INFO:teuthology.orchestra.run.smithi191.stdout:check_obsoletes = 1 2023-09-22T12:22:38.846 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum clean all 2023-09-22T12:22:38.864 DEBUG:teuthology.orchestra.run.smithi170:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-22T12:22:38.944 DEBUG:teuthology.orchestra.run.smithi170:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-22T12:22:39.011 INFO:teuthology.orchestra.run.smithi170.stdout:check_obsoletes = 1 2023-09-22T12:22:39.013 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum clean all 2023-09-22T12:22:39.110 INFO:teuthology.orchestra.run.smithi191.stdout:76 files removed 2023-09-22T12:22:39.138 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-radosgw 2023-09-22T12:22:39.286 INFO:teuthology.orchestra.run.smithi170.stdout:76 files removed 2023-09-22T12:22:39.312 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install ceph-radosgw 2023-09-22T12:22:39.886 INFO:teuthology.orchestra.run.smithi191.stdout:ceph packages for x86_64 222 kB/s | 76 kB 00:00 2023-09-22T12:22:40.101 INFO:teuthology.orchestra.run.smithi170.stdout:ceph packages for x86_64 200 kB/s | 76 kB 00:00 2023-09-22T12:22:40.262 INFO:teuthology.orchestra.run.smithi191.stdout:ceph noarch packages 45 kB/s | 15 kB 00:00 2023-09-22T12:22:40.417 INFO:teuthology.orchestra.run.smithi170.stdout:ceph noarch packages 55 kB/s | 15 kB 00:00 2023-09-22T12:22:40.563 INFO:teuthology.orchestra.run.smithi191.stdout:ceph source packages 5.2 kB/s | 1.4 kB 00:00 2023-09-22T12:22:40.718 INFO:teuthology.orchestra.run.smithi170.stdout:ceph source packages 5.3 kB/s | 1.4 kB 00:00 2023-09-22T12:39:56.280 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 8 - AppStream 33 kB/s | 33 MB 17:15 2023-09-22T12:40:09.402 INFO:teuthology.orchestra.run.smithi170.stdout:CentOS Stream 8 - AppStream 33 kB/s | 33 MB 17:28 2023-09-22T12:40:31.750 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 8 - BaseOS 1.6 MB/s | 47 MB 00:29 2023-09-22T12:40:31.874 INFO:teuthology.orchestra.run.smithi170.stdout:CentOS Stream 8 - BaseOS 2.9 MB/s | 47 MB 00:16 2023-09-22T12:40:39.008 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 8 - Extras 38 kB/s | 18 kB 00:00 2023-09-22T12:40:39.409 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 8 - Extras common packages 30 kB/s | 6.9 kB 00:00 2023-09-22T12:40:40.041 INFO:teuthology.orchestra.run.smithi170.stdout:CentOS Stream 8 - Extras 12 kB/s | 18 kB 00:01 2023-09-22T12:40:40.424 INFO:teuthology.orchestra.run.smithi170.stdout:CentOS Stream 8 - Extras common packages 34 kB/s | 6.9 kB 00:00 2023-09-22T12:40:41.637 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 8 - PowerTools 3.1 MB/s | 6.5 MB 00:02 2023-09-22T12:40:42.334 INFO:teuthology.orchestra.run.smithi170.stdout:CentOS Stream 8 - PowerTools 3.7 MB/s | 6.5 MB 00:01 2023-09-22T12:40:43.078 INFO:teuthology.orchestra.run.smithi191.stdout:Copr repo for python3-asyncssh owned by ceph 22 kB/s | 3.5 kB 00:00 2023-09-22T12:40:43.820 INFO:teuthology.orchestra.run.smithi170.stdout:Copr repo for python3-asyncssh owned by ceph 17 kB/s | 3.5 kB 00:00 2023-09-22T12:40:44.761 INFO:teuthology.orchestra.run.smithi170.stdout:Extra Packages for Enterprise Linux 20 MB/s | 16 MB 00:00 2023-09-22T12:40:45.165 INFO:teuthology.orchestra.run.smithi191.stdout:Extra Packages for Enterprise Linux 8.2 MB/s | 16 MB 00:01 2023-09-22T12:40:48.829 INFO:teuthology.orchestra.run.smithi170.stdout:lab-extras 128 kB/s | 24 kB 00:00 2023-09-22T12:40:48.962 INFO:teuthology.orchestra.run.smithi191.stdout:lab-extras 381 kB/s | 24 kB 00:00 2023-09-22T12:40:53.240 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:40:53.241 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:40:53.241 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repository Size 2023-09-22T12:40:53.242 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:40:53.242 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T12:40:53.242 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-radosgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 12 M 2023-09-22T12:40:53.242 INFO:teuthology.orchestra.run.smithi170.stdout:Upgrading: 2023-09-22T12:40:53.242 INFO:teuthology.orchestra.run.smithi170.stdout: librados2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T12:40:53.242 INFO:teuthology.orchestra.run.smithi170.stdout: librbd1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 4.2 M 2023-09-22T12:40:53.243 INFO:teuthology.orchestra.run.smithi170.stdout:Installing dependencies: 2023-09-22T12:40:53.243 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-base x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.1 M 2023-09-22T12:40:53.243 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 24 M 2023-09-22T12:40:53.243 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-selinux x86_64 2:16.2.14-5.g31c126b7.el8 ceph 26 k 2023-09-22T12:40:53.243 INFO:teuthology.orchestra.run.smithi170.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-22T12:40:53.243 INFO:teuthology.orchestra.run.smithi170.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-09-22T12:40:53.243 INFO:teuthology.orchestra.run.smithi170.stdout: libcephfs2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 824 k 2023-09-22T12:40:53.244 INFO:teuthology.orchestra.run.smithi170.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-22T12:40:53.244 INFO:teuthology.orchestra.run.smithi170.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-09-22T12:40:53.244 INFO:teuthology.orchestra.run.smithi170.stdout: libradosstriper1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 494 k 2023-09-22T12:40:53.244 INFO:teuthology.orchestra.run.smithi170.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-09-22T12:40:53.244 INFO:teuthology.orchestra.run.smithi170.stdout: librgw2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.8 M 2023-09-22T12:40:53.244 INFO:teuthology.orchestra.run.smithi170.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-22T12:40:53.244 INFO:teuthology.orchestra.run.smithi170.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-09-22T12:40:53.244 INFO:teuthology.orchestra.run.smithi170.stdout: python3-ceph-argparse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 45 k 2023-09-22T12:40:53.245 INFO:teuthology.orchestra.run.smithi170.stdout: python3-ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 99 k 2023-09-22T12:40:53.245 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cephfs x86_64 2:16.2.14-5.g31c126b7.el8 ceph 215 k 2023-09-22T12:40:53.245 INFO:teuthology.orchestra.run.smithi170.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-09-22T12:40:53.245 INFO:teuthology.orchestra.run.smithi170.stdout: python3-rados x86_64 2:16.2.14-5.g31c126b7.el8 ceph 390 k 2023-09-22T12:40:53.245 INFO:teuthology.orchestra.run.smithi170.stdout: python3-rbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 369 k 2023-09-22T12:40:53.245 INFO:teuthology.orchestra.run.smithi170.stdout: python3-rgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 114 k 2023-09-22T12:40:53.245 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:40:53.245 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T12:40:53.246 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:40:53.246 INFO:teuthology.orchestra.run.smithi170.stdout:Install 21 Packages 2023-09-22T12:40:53.246 INFO:teuthology.orchestra.run.smithi170.stdout:Upgrade 2 Packages 2023-09-22T12:40:53.246 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:40:53.247 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 57 M 2023-09-22T12:40:53.247 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T12:40:53.380 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T12:40:53.382 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:40:53.382 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-09-22T12:40:53.382 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:40:53.382 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T12:40:53.383 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-radosgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 12 M 2023-09-22T12:40:53.383 INFO:teuthology.orchestra.run.smithi191.stdout:Upgrading: 2023-09-22T12:40:53.383 INFO:teuthology.orchestra.run.smithi191.stdout: librados2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T12:40:53.383 INFO:teuthology.orchestra.run.smithi191.stdout: librbd1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 4.2 M 2023-09-22T12:40:53.383 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-09-22T12:40:53.383 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-base x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.1 M 2023-09-22T12:40:53.383 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 24 M 2023-09-22T12:40:53.383 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-selinux x86_64 2:16.2.14-5.g31c126b7.el8 ceph 26 k 2023-09-22T12:40:53.384 INFO:teuthology.orchestra.run.smithi191.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-22T12:40:53.384 INFO:teuthology.orchestra.run.smithi191.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-09-22T12:40:53.384 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 824 k 2023-09-22T12:40:53.384 INFO:teuthology.orchestra.run.smithi191.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-22T12:40:53.384 INFO:teuthology.orchestra.run.smithi191.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-09-22T12:40:53.384 INFO:teuthology.orchestra.run.smithi191.stdout: libradosstriper1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 494 k 2023-09-22T12:40:53.384 INFO:teuthology.orchestra.run.smithi191.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-09-22T12:40:53.384 INFO:teuthology.orchestra.run.smithi191.stdout: librgw2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.8 M 2023-09-22T12:40:53.384 INFO:teuthology.orchestra.run.smithi191.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-22T12:40:53.385 INFO:teuthology.orchestra.run.smithi191.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-09-22T12:40:53.385 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-argparse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 45 k 2023-09-22T12:40:53.385 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 99 k 2023-09-22T12:40:53.385 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cephfs x86_64 2:16.2.14-5.g31c126b7.el8 ceph 215 k 2023-09-22T12:40:53.385 INFO:teuthology.orchestra.run.smithi191.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-09-22T12:40:53.385 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rados x86_64 2:16.2.14-5.g31c126b7.el8 ceph 390 k 2023-09-22T12:40:53.385 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 369 k 2023-09-22T12:40:53.385 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 114 k 2023-09-22T12:40:53.385 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:40:53.386 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T12:40:53.386 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:40:53.386 INFO:teuthology.orchestra.run.smithi191.stdout:Install 21 Packages 2023-09-22T12:40:53.386 INFO:teuthology.orchestra.run.smithi191.stdout:Upgrade 2 Packages 2023-09-22T12:40:53.386 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:40:53.387 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 57 M 2023-09-22T12:40:53.387 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T12:40:56.095 INFO:teuthology.orchestra.run.smithi191.stdout:(1/23): ceph-radosgw-16.2.14-5.g31c126b7.el8.x8 5.2 MB/s | 12 MB 00:02 2023-09-22T12:40:56.187 INFO:teuthology.orchestra.run.smithi191.stdout:(2/23): ceph-selinux-16.2.14-5.g31c126b7.el8.x8 281 kB/s | 26 kB 00:00 2023-09-22T12:40:56.454 INFO:teuthology.orchestra.run.smithi191.stdout:(3/23): libcephfs2-16.2.14-5.g31c126b7.el8.x86_ 3.0 MB/s | 824 kB 00:00 2023-09-22T12:40:56.629 INFO:teuthology.orchestra.run.smithi191.stdout:(4/23): libradosstriper1-16.2.14-5.g31c126b7.el 2.8 MB/s | 494 kB 00:00 2023-09-22T12:40:56.637 INFO:teuthology.orchestra.run.smithi170.stdout:(1/23): ceph-base-16.2.14-5.g31c126b7.el8.x86_6 2.0 MB/s | 6.1 MB 00:03 2023-09-22T12:40:56.762 INFO:teuthology.orchestra.run.smithi170.stdout:(2/23): ceph-selinux-16.2.14-5.g31c126b7.el8.x8 205 kB/s | 26 kB 00:00 2023-09-22T12:40:57.213 INFO:teuthology.orchestra.run.smithi170.stdout:(3/23): libcephfs2-16.2.14-5.g31c126b7.el8.x86_ 1.8 MB/s | 824 kB 00:00 2023-09-22T12:40:57.448 INFO:teuthology.orchestra.run.smithi170.stdout:(4/23): libradosstriper1-16.2.14-5.g31c126b7.el 2.1 MB/s | 494 kB 00:00 2023-09-22T12:40:57.530 INFO:teuthology.orchestra.run.smithi191.stdout:(5/23): ceph-common-16.2.14-5.g31c126b7.el8.x86 6.6 MB/s | 24 MB 00:03 2023-09-22T12:40:57.630 INFO:teuthology.orchestra.run.smithi191.stdout:(6/23): librgw2-16.2.14-5.g31c126b7.el8.x86_64. 3.8 MB/s | 3.8 MB 00:01 2023-09-22T12:40:57.647 INFO:teuthology.orchestra.run.smithi191.stdout:(7/23): python3-ceph-argparse-16.2.14-5.g31c126 386 kB/s | 45 kB 00:00 2023-09-22T12:40:57.697 INFO:teuthology.orchestra.run.smithi191.stdout:(8/23): python3-ceph-common-16.2.14-5.g31c126b7 1.4 MB/s | 99 kB 00:00 2023-09-22T12:40:57.722 INFO:teuthology.orchestra.run.smithi191.stdout:(9/23): python3-cephfs-16.2.14-5.g31c126b7.el8. 2.8 MB/s | 215 kB 00:00 2023-09-22T12:40:57.814 INFO:teuthology.orchestra.run.smithi191.stdout:(10/23): python3-rbd-16.2.14-5.g31c126b7.el8.x8 3.9 MB/s | 369 kB 00:00 2023-09-22T12:40:57.839 INFO:teuthology.orchestra.run.smithi191.stdout:(11/23): python3-rados-16.2.14-5.g31c126b7.el8. 2.7 MB/s | 390 kB 00:00 2023-09-22T12:40:57.891 INFO:teuthology.orchestra.run.smithi191.stdout:(12/23): python3-rgw-16.2.14-5.g31c126b7.el8.x8 1.5 MB/s | 114 kB 00:00 2023-09-22T12:40:58.854 INFO:teuthology.orchestra.run.smithi170.stdout:(5/23): librgw2-16.2.14-5.g31c126b7.el8.x86_64. 2.7 MB/s | 3.8 MB 00:01 2023-09-22T12:40:58.946 INFO:teuthology.orchestra.run.smithi170.stdout:(6/23): python3-ceph-argparse-16.2.14-5.g31c126 491 kB/s | 45 kB 00:00 2023-09-22T12:40:59.071 INFO:teuthology.orchestra.run.smithi170.stdout:(7/23): python3-ceph-common-16.2.14-5.g31c126b7 789 kB/s | 99 kB 00:00 2023-09-22T12:40:59.189 INFO:teuthology.orchestra.run.smithi170.stdout:(8/23): python3-cephfs-16.2.14-5.g31c126b7.el8. 1.8 MB/s | 215 kB 00:00 2023-09-22T12:40:59.364 INFO:teuthology.orchestra.run.smithi170.stdout:(9/23): python3-rados-16.2.14-5.g31c126b7.el8.x 2.2 MB/s | 390 kB 00:00 2023-09-22T12:40:59.547 INFO:teuthology.orchestra.run.smithi170.stdout:(10/23): python3-rbd-16.2.14-5.g31c126b7.el8.x8 2.0 MB/s | 369 kB 00:00 2023-09-22T12:40:59.639 INFO:teuthology.orchestra.run.smithi170.stdout:(11/23): python3-rgw-16.2.14-5.g31c126b7.el8.x8 1.2 MB/s | 114 kB 00:00 2023-09-22T12:41:00.865 INFO:teuthology.orchestra.run.smithi170.stdout:(12/23): ceph-radosgw-16.2.14-5.g31c126b7.el8.x 1.6 MB/s | 12 MB 00:07 2023-09-22T12:41:01.282 INFO:teuthology.orchestra.run.smithi191.stdout:(13/23): ceph-base-16.2.14-5.g31c126b7.el8.x86_ 839 kB/s | 6.1 MB 00:07 2023-09-22T12:41:02.032 INFO:teuthology.orchestra.run.smithi191.stdout:(14/23): python3-prettytable-0.7.2-14.el8.noarc 59 kB/s | 44 kB 00:00 2023-09-22T12:41:02.357 INFO:teuthology.orchestra.run.smithi191.stdout:(15/23): librabbitmq-0.9.0-4.el8.x86_64.rpm 144 kB/s | 47 kB 00:00 2023-09-22T12:41:04.407 INFO:teuthology.orchestra.run.smithi191.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 149 kB/s | 306 kB 00:02 2023-09-22T12:41:04.516 INFO:teuthology.orchestra.run.smithi191.stdout:(17/23): leveldb-1.22-1.el8.x86_64.rpm 1.6 MB/s | 181 kB 00:00 2023-09-22T12:41:04.574 INFO:teuthology.orchestra.run.smithi191.stdout:(18/23): liboath-2.6.2-3.el8.x86_64.rpm 1.0 MB/s | 59 kB 00:00 2023-09-22T12:41:04.699 INFO:teuthology.orchestra.run.smithi191.stdout:(19/23): libunwind-1.3.1-3.el8.x86_64.rpm 601 kB/s | 75 kB 00:00 2023-09-22T12:41:05.356 INFO:teuthology.orchestra.run.smithi170.stdout:(13/23): ceph-common-16.2.14-5.g31c126b7.el8.x8 2.1 MB/s | 24 MB 00:11 2023-09-22T12:41:06.358 INFO:teuthology.orchestra.run.smithi191.stdout:(20/23): librados2-16.2.14-5.g31c126b7.el8.x86_ 2.3 MB/s | 3.7 MB 00:01 2023-09-22T12:41:06.689 INFO:teuthology.orchestra.run.smithi170.stdout:(14/23): python3-prettytable-0.7.2-14.el8.noarc 33 kB/s | 44 kB 00:01 2023-09-22T12:41:06.981 INFO:teuthology.orchestra.run.smithi170.stdout:(15/23): librabbitmq-0.9.0-4.el8.x86_64.rpm 161 kB/s | 47 kB 00:00 2023-09-22T12:41:07.525 INFO:teuthology.orchestra.run.smithi191.stdout:(21/23): librbd1-16.2.14-5.g31c126b7.el8.x86_64 3.6 MB/s | 4.2 MB 00:01 2023-09-22T12:41:07.756 INFO:teuthology.orchestra.run.smithi170.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 395 kB/s | 306 kB 00:00 2023-09-22T12:41:07.875 INFO:teuthology.orchestra.run.smithi191.stdout:(22/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 26 kB/s | 259 kB 00:09 2023-09-22T12:41:07.923 INFO:teuthology.orchestra.run.smithi170.stdout:(17/23): leveldb-1.22-1.el8.x86_64.rpm 1.1 MB/s | 181 kB 00:00 2023-09-22T12:41:08.057 INFO:teuthology.orchestra.run.smithi170.stdout:(18/23): liboath-2.6.2-3.el8.x86_64.rpm 441 kB/s | 59 kB 00:00 2023-09-22T12:41:08.357 INFO:teuthology.orchestra.run.smithi170.stdout:(19/23): libunwind-1.3.1-3.el8.x86_64.rpm 250 kB/s | 75 kB 00:00 2023-09-22T12:41:09.116 INFO:teuthology.orchestra.run.smithi170.stdout:(20/23): librados2-16.2.14-5.g31c126b7.el8.x86_ 4.9 MB/s | 3.7 MB 00:00 2023-09-22T12:41:09.808 INFO:teuthology.orchestra.run.smithi170.stdout:(21/23): librbd1-16.2.14-5.g31c126b7.el8.x86_64 6.0 MB/s | 4.2 MB 00:00 2023-09-22T12:41:10.467 INFO:teuthology.orchestra.run.smithi170.stdout:(22/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 27 kB/s | 259 kB 00:09 2023-09-22T12:41:11.300 INFO:teuthology.orchestra.run.smithi191.stdout:(23/23): librdkafka-0.11.4-3.el8.x86_64.rpm 26 kB/s | 354 kB 00:13 2023-09-22T12:41:11.306 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:41:11.306 INFO:teuthology.orchestra.run.smithi191.stdout:Total 3.2 MB/s | 57 MB 00:17 2023-09-22T12:41:11.343 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T12:41:11.390 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T12:41:11.391 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T12:41:11.752 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T12:41:11.755 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T12:41:12.782 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T12:41:12.983 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2023-09-22T12:41:13.146 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T12:41:13.199 INFO:teuthology.orchestra.run.smithi170.stdout:(23/23): librdkafka-0.11.4-3.el8.x86_64.rpm 26 kB/s | 354 kB 00:13 2023-09-22T12:41:13.203 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:41:13.203 INFO:teuthology.orchestra.run.smithi170.stdout:Total 2.9 MB/s | 57 MB 00:19 2023-09-22T12:41:13.243 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T12:41:13.290 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T12:41:13.290 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T12:41:13.531 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T12:41:13.630 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T12:41:13.633 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T12:41:13.697 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:41:13.791 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:41:14.120 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T12:41:14.389 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:41:14.783 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T12:41:14.798 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:41:14.974 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2023-09-22T12:41:14.979 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:41:15.029 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:41:15.156 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T12:41:15.227 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 7/25 2023-09-22T12:41:15.388 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T12:41:15.540 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T12:41:15.714 INFO:teuthology.orchestra.run.smithi170.stdout: Upgrading : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:41:15.764 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T12:41:15.808 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:41:15.904 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T12:41:15.961 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T12:41:16.063 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T12:41:16.126 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-09-22T12:41:16.222 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:41:16.268 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 11/25 2023-09-22T12:41:16.443 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T12:41:16.617 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:41:16.689 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T12:41:16.780 INFO:teuthology.orchestra.run.smithi170.stdout: Upgrading : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:41:16.855 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:41:16.896 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 14/25 2023-09-22T12:41:17.028 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 7/25 2023-09-22T12:41:17.046 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T12:41:17.116 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T12:41:17.205 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T12:41:17.329 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-22T12:41:17.512 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2023-09-22T12:41:17.572 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T12:41:17.690 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2023-09-22T12:41:17.713 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T12:41:17.796 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T12:41:17.888 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 19/25 2023-09-22T12:41:17.977 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-09-22T12:41:18.135 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 11/25 2023-09-22T12:41:18.302 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T12:41:18.506 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T12:41:18.722 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 14/25 2023-09-22T12:41:18.847 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T12:41:18.935 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T12:41:19.380 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-22T12:41:19.554 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2023-09-22T12:41:19.724 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2023-09-22T12:41:19.906 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 19/25 2023-09-22T12:41:20.484 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:41:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:41:21.432 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:41:21.696 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T12:41:21.801 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T12:41:21.920 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:41:22.428 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:41:22.638 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:41:23.380 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:41:23.650 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T12:41:23.753 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T12:41:23.879 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:41:39.918 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:41:39.919 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /sys 2023-09-22T12:41:39.919 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /proc 2023-09-22T12:41:39.919 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /dev 2023-09-22T12:41:39.919 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /run 2023-09-22T12:41:39.919 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /mnt 2023-09-22T12:41:39.919 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /var/tmp 2023-09-22T12:41:39.919 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /home 2023-09-22T12:41:39.919 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /tmp 2023-09-22T12:41:39.919 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /dev 2023-09-22T12:41:39.919 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:41:41.084 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T12:41:41.161 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T12:41:41.161 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T12:41:41.368 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T12:41:41.368 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:41:41.991 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:41:41.992 INFO:teuthology.orchestra.run.smithi170.stdout:skipping the directory /sys 2023-09-22T12:41:41.992 INFO:teuthology.orchestra.run.smithi170.stdout:skipping the directory /proc 2023-09-22T12:41:41.992 INFO:teuthology.orchestra.run.smithi170.stdout:skipping the directory /dev 2023-09-22T12:41:41.992 INFO:teuthology.orchestra.run.smithi170.stdout:skipping the directory /run 2023-09-22T12:41:41.992 INFO:teuthology.orchestra.run.smithi170.stdout:skipping the directory /mnt 2023-09-22T12:41:41.992 INFO:teuthology.orchestra.run.smithi170.stdout:skipping the directory /var/tmp 2023-09-22T12:41:41.992 INFO:teuthology.orchestra.run.smithi170.stdout:skipping the directory /home 2023-09-22T12:41:41.992 INFO:teuthology.orchestra.run.smithi170.stdout:skipping the directory /tmp 2023-09-22T12:41:41.992 INFO:teuthology.orchestra.run.smithi170.stdout:skipping the directory /dev 2023-09-22T12:41:41.992 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:41:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:41:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 1/25 2023-09-22T12:41:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2/25 2023-09-22T12:41:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:41:42.042 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 7/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 8/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 9/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 10/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 11/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2023-09-22T12:41:42.043 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-22T12:41:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 17/25 2023-09-22T12:41:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-22T12:41:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : leveldb-1.22-1.el8.x86_64 19/25 2023-09-22T12:41:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 20/25 2023-09-22T12:41:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 21/25 2023-09-22T12:41:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:41:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-22T12:41:42.044 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 24/25 2023-09-22T12:41:43.130 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T12:41:43.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:41:43.172 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:41:43.172 INFO:teuthology.orchestra.run.smithi191.stdout:Upgraded: 2023-09-22T12:41:43.172 INFO:teuthology.orchestra.run.smithi191.stdout: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.172 INFO:teuthology.orchestra.run.smithi191.stdout: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.172 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T12:41:43.172 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.173 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.173 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.173 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.173 INFO:teuthology.orchestra.run.smithi191.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-22T12:41:43.173 INFO:teuthology.orchestra.run.smithi191.stdout: leveldb-1.22-1.el8.x86_64 2023-09-22T12:41:43.173 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.173 INFO:teuthology.orchestra.run.smithi191.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-22T12:41:43.173 INFO:teuthology.orchestra.run.smithi191.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-09-22T12:41:43.173 INFO:teuthology.orchestra.run.smithi191.stdout: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.173 INFO:teuthology.orchestra.run.smithi191.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-22T12:41:43.173 INFO:teuthology.orchestra.run.smithi191.stdout: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.174 INFO:teuthology.orchestra.run.smithi191.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-22T12:41:43.174 INFO:teuthology.orchestra.run.smithi191.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-22T12:41:43.174 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.174 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.174 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.174 INFO:teuthology.orchestra.run.smithi191.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-22T12:41:43.174 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.174 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.174 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:43.174 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:41:43.174 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T12:41:43.205 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T12:41:43.205 INFO:teuthology.orchestra.run.smithi170.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T12:41:43.403 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T12:41:43.403 INFO:teuthology.orchestra.run.smithi170.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:41:43.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-test 2023-09-22T12:41:44.105 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:41:44.105 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 1/25 2023-09-22T12:41:44.106 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2/25 2023-09-22T12:41:44.106 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:41:44.106 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T12:41:44.106 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:41:44.106 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:41:44.106 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 7/25 2023-09-22T12:41:44.106 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 8/25 2023-09-22T12:41:44.106 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 9/25 2023-09-22T12:41:44.106 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 10/25 2023-09-22T12:41:44.106 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 11/25 2023-09-22T12:41:44.106 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 17/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : leveldb-1.22-1.el8.x86_64 19/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 20/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 21/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-22T12:41:44.107 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 24/25 2023-09-22T12:41:44.274 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:56 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:41:45.136 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:41:45.136 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:41:45.136 INFO:teuthology.orchestra.run.smithi170.stdout:Upgraded: 2023-09-22T12:41:45.136 INFO:teuthology.orchestra.run.smithi170.stdout: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout: leveldb-1.22-1.el8.x86_64 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-09-22T12:41:45.137 INFO:teuthology.orchestra.run.smithi170.stdout: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.138 INFO:teuthology.orchestra.run.smithi170.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-22T12:41:45.138 INFO:teuthology.orchestra.run.smithi170.stdout: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.138 INFO:teuthology.orchestra.run.smithi170.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-22T12:41:45.138 INFO:teuthology.orchestra.run.smithi170.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-22T12:41:45.138 INFO:teuthology.orchestra.run.smithi170.stdout: python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.138 INFO:teuthology.orchestra.run.smithi170.stdout: python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.138 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.138 INFO:teuthology.orchestra.run.smithi170.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-22T12:41:45.138 INFO:teuthology.orchestra.run.smithi170.stdout: python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.138 INFO:teuthology.orchestra.run.smithi170.stdout: python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.138 INFO:teuthology.orchestra.run.smithi170.stdout: python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:41:45.139 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:41:45.139 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T12:41:45.437 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install ceph-test 2023-09-22T12:41:45.847 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T12:41:45.848 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:41:45.848 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-09-22T12:41:45.848 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:41:45.849 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T12:41:45.849 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-test x86_64 2:16.2.14-5.g31c126b7.el8 ceph 52 M 2023-09-22T12:41:45.849 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-09-22T12:41:45.849 INFO:teuthology.orchestra.run.smithi191.stdout: jq x86_64 1.6-7.el8 appstream 202 k 2023-09-22T12:41:45.849 INFO:teuthology.orchestra.run.smithi191.stdout: libcephsqlite x86_64 2:16.2.14-5.g31c126b7.el8 ceph 174 k 2023-09-22T12:41:45.849 INFO:teuthology.orchestra.run.smithi191.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-09-22T12:41:45.849 INFO:teuthology.orchestra.run.smithi191.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-09-22T12:41:45.849 INFO:teuthology.orchestra.run.smithi191.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-09-22T12:41:45.850 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:41:45.850 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T12:41:45.850 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:41:45.850 INFO:teuthology.orchestra.run.smithi191.stdout:Install 6 Packages 2023-09-22T12:41:45.850 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:41:45.851 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 53 M 2023-09-22T12:41:45.851 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 196 M 2023-09-22T12:41:45.851 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T12:41:46.185 INFO:teuthology.orchestra.run.smithi191.stdout:(1/6): libcephsqlite-16.2.14-5.g31c126b7.el8.x8 594 kB/s | 174 kB 00:00 2023-09-22T12:41:46.288 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:00:58 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:41:47.860 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:41:47.861 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:41:47.862 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repository Size 2023-09-22T12:41:47.862 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:41:47.862 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T12:41:47.862 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-test x86_64 2:16.2.14-5.g31c126b7.el8 ceph 52 M 2023-09-22T12:41:47.862 INFO:teuthology.orchestra.run.smithi170.stdout:Installing dependencies: 2023-09-22T12:41:47.862 INFO:teuthology.orchestra.run.smithi170.stdout: jq x86_64 1.6-7.el8 appstream 202 k 2023-09-22T12:41:47.863 INFO:teuthology.orchestra.run.smithi170.stdout: libcephsqlite x86_64 2:16.2.14-5.g31c126b7.el8 ceph 174 k 2023-09-22T12:41:47.863 INFO:teuthology.orchestra.run.smithi170.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-09-22T12:41:47.863 INFO:teuthology.orchestra.run.smithi170.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-09-22T12:41:47.863 INFO:teuthology.orchestra.run.smithi170.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-09-22T12:41:47.863 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:41:47.863 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T12:41:47.863 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:41:47.863 INFO:teuthology.orchestra.run.smithi170.stdout:Install 6 Packages 2023-09-22T12:41:47.864 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:41:47.864 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 53 M 2023-09-22T12:41:47.865 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 196 M 2023-09-22T12:41:47.865 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T12:41:48.123 INFO:teuthology.orchestra.run.smithi170.stdout:(1/6): libcephsqlite-16.2.14-5.g31c126b7.el8.x8 797 kB/s | 174 kB 00:00 2023-09-22T12:41:48.427 INFO:teuthology.orchestra.run.smithi191.stdout:(2/6): ceph-test-16.2.14-5.g31c126b7.el8.x86_64 20 MB/s | 52 MB 00:02 2023-09-22T12:41:51.324 INFO:teuthology.orchestra.run.smithi170.stdout:(2/6): ceph-test-16.2.14-5.g31c126b7.el8.x86_64 15 MB/s | 52 MB 00:03 2023-09-22T12:41:53.609 INFO:teuthology.orchestra.run.smithi191.stdout:(3/6): jq-1.6-7.el8.x86_64.rpm 26 kB/s | 202 kB 00:07 2023-09-22T12:41:53.665 INFO:teuthology.orchestra.run.smithi170.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 34 kB/s | 187 kB 00:05 2023-09-22T12:41:55.907 INFO:teuthology.orchestra.run.smithi170.stdout:(4/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 31 kB/s | 71 kB 00:02 2023-09-22T12:41:56.117 INFO:teuthology.orchestra.run.smithi191.stdout:(4/6): oniguruma-6.8.2-2.el8.x86_64.rpm 19 kB/s | 187 kB 00:09 2023-09-22T12:41:56.265 INFO:teuthology.orchestra.run.smithi170.stdout:(5/6): jq-1.6-7.el8.x86_64.rpm 24 kB/s | 202 kB 00:08 2023-09-22T12:41:59.651 INFO:teuthology.orchestra.run.smithi191.stdout:(5/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 12 kB/s | 71 kB 00:06 2023-09-22T12:42:04.042 INFO:teuthology.orchestra.run.smithi191.stdout:(6/6): socat-1.7.4.1-1.el8.x86_64.rpm 21 kB/s | 323 kB 00:15 2023-09-22T12:42:04.043 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:42:04.043 INFO:teuthology.orchestra.run.smithi191.stdout:Total 2.9 MB/s | 53 MB 00:18 2023-09-22T12:42:04.086 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T12:42:04.107 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T12:42:04.107 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T12:42:04.421 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T12:42:04.424 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T12:42:04.905 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T12:42:05.177 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-09-22T12:42:05.381 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-09-22T12:42:05.495 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T12:42:05.572 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T12:42:05.751 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : jq-1.6-7.el8.x86_64 4/6 2023-09-22T12:42:05.903 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T12:42:10.535 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T12:42:10.776 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T12:42:11.071 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T12:42:11.071 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 1/6 2023-09-22T12:42:11.071 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 2/6 2023-09-22T12:42:11.072 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : jq-1.6-7.el8.x86_64 3/6 2023-09-22T12:42:11.072 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-09-22T12:42:11.072 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-09-22T12:42:11.704 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-09-22T12:42:11.704 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:11.705 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T12:42:11.705 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 jq-1.6-7.el8.x86_64 2023-09-22T12:42:11.705 INFO:teuthology.orchestra.run.smithi191.stdout: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 oniguruma-6.8.2-2.el8.x86_64 2023-09-22T12:42:11.705 INFO:teuthology.orchestra.run.smithi191.stdout: socat-1.7.4.1-1.el8.x86_64 xmlstarlet-1.6.1-20.el8.x86_64 2023-09-22T12:42:11.705 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:11.705 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T12:42:11.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph 2023-09-22T12:42:12.802 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:24 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:42:14.370 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T12:42:14.373 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:42:14.374 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-09-22T12:42:14.374 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:42:14.374 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T12:42:14.374 INFO:teuthology.orchestra.run.smithi191.stdout: ceph x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.4 k 2023-09-22T12:42:14.374 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-09-22T12:42:14.374 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mds x86_64 2:16.2.14-5.g31c126b7.el8 ceph 2.4 M 2023-09-22T12:42:14.374 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr x86_64 2:16.2.14-5.g31c126b7.el8 ceph 1.6 M 2023-09-22T12:42:14.374 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-modules-core noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 244 k 2023-09-22T12:42:14.374 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mon x86_64 2:16.2.14-5.g31c126b7.el8 ceph 5.0 M 2023-09-22T12:42:14.374 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-osd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 18 M 2023-09-22T12:42:14.375 INFO:teuthology.orchestra.run.smithi191.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-22T12:42:14.375 INFO:teuthology.orchestra.run.smithi191.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-22T12:42:14.375 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-09-22T12:42:14.375 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-22T12:42:14.375 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-22T12:42:14.375 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-09-22T12:42:14.375 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-22T12:42:14.375 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-22T12:42:14.375 INFO:teuthology.orchestra.run.smithi191.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-22T12:42:14.375 INFO:teuthology.orchestra.run.smithi191.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-09-22T12:42:14.375 INFO:teuthology.orchestra.run.smithi191.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-22T12:42:14.376 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-22T12:42:14.377 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-22T12:42:14.377 INFO:teuthology.orchestra.run.smithi191.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-09-22T12:42:14.377 INFO:teuthology.orchestra.run.smithi191.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-22T12:42:14.377 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:14.377 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T12:42:14.377 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:42:14.377 INFO:teuthology.orchestra.run.smithi191.stdout:Install 32 Packages 2023-09-22T12:42:14.377 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:14.378 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 31 M 2023-09-22T12:42:14.378 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 101 M 2023-09-22T12:42:14.378 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T12:42:15.159 INFO:teuthology.orchestra.run.smithi191.stdout:(1/32): ceph-16.2.14-5.g31c126b7.el8.x86_64.rpm 33 kB/s | 6.4 kB 00:00 2023-09-22T12:42:15.309 INFO:teuthology.orchestra.run.smithi191.stdout:(2/32): ceph-mgr-16.2.14-5.g31c126b7.el8.x86_64 4.7 MB/s | 1.6 MB 00:00 2023-09-22T12:42:15.768 INFO:teuthology.orchestra.run.smithi191.stdout:(3/32): ceph-mds-16.2.14-5.g31c126b7.el8.x86_64 3.0 MB/s | 2.4 MB 00:00 2023-09-22T12:42:15.868 INFO:teuthology.orchestra.run.smithi191.stdout:(4/32): ceph-mgr-modules-core-16.2.14-5.g31c126 2.4 MB/s | 244 kB 00:00 2023-09-22T12:42:16.460 INFO:teuthology.orchestra.run.smithi191.stdout:(5/32): ceph-osd-16.2.14-5.g31c126b7.el8.x86_64 16 MB/s | 18 MB 00:01 2023-09-22T12:42:16.470 INFO:teuthology.orchestra.run.smithi170.stdout:(6/6): socat-1.7.4.1-1.el8.x86_64.rpm 13 kB/s | 323 kB 00:25 2023-09-22T12:42:16.471 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:42:16.472 INFO:teuthology.orchestra.run.smithi170.stdout:Total 1.8 MB/s | 53 MB 00:28 2023-09-22T12:42:16.515 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T12:42:16.527 INFO:teuthology.orchestra.run.smithi191.stdout:(6/32): ceph-mon-16.2.14-5.g31c126b7.el8.x86_64 3.7 MB/s | 5.0 MB 00:01 2023-09-22T12:42:16.535 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T12:42:16.535 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T12:42:16.909 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T12:42:16.912 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T12:42:17.452 INFO:teuthology.orchestra.run.smithi191.stdout:(7/32): python3-markupsafe-0.23-19.el8.x86_64.r 39 kB/s | 39 kB 00:00 2023-09-22T12:42:17.712 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T12:42:17.962 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-09-22T12:42:18.156 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-09-22T12:42:18.304 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T12:42:18.336 INFO:teuthology.orchestra.run.smithi191.stdout:(8/32): python3-pytz-2017.2-11.el8.noarch.rpm 65 kB/s | 57 kB 00:00 2023-09-22T12:42:18.384 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T12:42:18.576 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : jq-1.6-7.el8.x86_64 4/6 2023-09-22T12:42:18.653 INFO:teuthology.orchestra.run.smithi191.stdout:(9/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 49 kB/s | 103 kB 00:02 2023-09-22T12:42:18.728 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T12:42:19.178 INFO:teuthology.orchestra.run.smithi191.stdout:(10/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 474 kB/s | 249 kB 00:00 2023-09-22T12:42:19.403 INFO:teuthology.orchestra.run.smithi191.stdout:(11/32): python3-cryptography-3.2.1-6.el8.x86_6 2.8 MB/s | 641 kB 00:00 2023-09-22T12:42:19.504 INFO:teuthology.orchestra.run.smithi191.stdout:(12/32): python3-pycparser-2.14-14.el8.noarch.r 1.1 MB/s | 109 kB 00:00 2023-09-22T12:42:19.770 INFO:teuthology.orchestra.run.smithi191.stdout:(13/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 164 kB/s | 44 kB 00:00 2023-09-22T12:42:19.946 INFO:teuthology.orchestra.run.smithi191.stdout:(14/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 1.0 MB/s | 185 kB 00:00 2023-09-22T12:42:20.029 INFO:teuthology.orchestra.run.smithi191.stdout:(15/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 2.0 MB/s | 173 kB 00:00 2023-09-22T12:42:20.121 INFO:teuthology.orchestra.run.smithi191.stdout:(16/32): python3-cherrypy-18.4.0-1.el8.noarch.r 4.1 MB/s | 384 kB 00:00 2023-09-22T12:42:20.188 INFO:teuthology.orchestra.run.smithi191.stdout:(17/32): python3-jaraco-6.2-6.el8.noarch.rpm 167 kB/s | 11 kB 00:00 2023-09-22T12:42:20.246 INFO:teuthology.orchestra.run.smithi191.stdout:(18/32): python3-jaraco-functools-2.0-4.el8.noa 302 kB/s | 18 kB 00:00 2023-09-22T12:42:20.314 INFO:teuthology.orchestra.run.smithi191.stdout:(19/32): python3-logutils-0.3.5-11.el8.noarch.r 730 kB/s | 49 kB 00:00 2023-09-22T12:42:20.372 INFO:teuthology.orchestra.run.smithi191.stdout:(20/32): python3-more-itertools-7.2.0-3.el8.noa 1.0 MB/s | 59 kB 00:00 2023-09-22T12:42:20.456 INFO:teuthology.orchestra.run.smithi191.stdout:(21/32): python3-pecan-1.3.2-9.el8.noarch.rpm 3.3 MB/s | 283 kB 00:00 2023-09-22T12:42:20.523 INFO:teuthology.orchestra.run.smithi191.stdout:(22/32): python3-portend-2.6-1.el8.noarch.rpm 234 kB/s | 16 kB 00:00 2023-09-22T12:42:20.598 INFO:teuthology.orchestra.run.smithi191.stdout:(23/32): python3-simplegeneric-0.8.1-17.el8.noa 249 kB/s | 19 kB 00:00 2023-09-22T12:42:20.673 INFO:teuthology.orchestra.run.smithi191.stdout:(24/32): python3-singledispatch-3.4.0.3-18.el8. 320 kB/s | 24 kB 00:00 2023-09-22T12:42:20.740 INFO:teuthology.orchestra.run.smithi191.stdout:(25/32): python3-tempora-1.14.1-5.el8.noarch.rp 438 kB/s | 29 kB 00:00 2023-09-22T12:42:20.807 INFO:teuthology.orchestra.run.smithi191.stdout:(26/32): python3-trustme-0.6.0-4.el8.noarch.rpm 413 kB/s | 27 kB 00:00 2023-09-22T12:42:20.882 INFO:teuthology.orchestra.run.smithi191.stdout:(27/32): python3-waitress-1.4.3-1.el8.noarch.rp 3.3 MB/s | 255 kB 00:00 2023-09-22T12:42:20.957 INFO:teuthology.orchestra.run.smithi191.stdout:(28/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 3.3 MB/s | 251 kB 00:00 2023-09-22T12:42:21.033 INFO:teuthology.orchestra.run.smithi191.stdout:(29/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.1 MB/s | 85 kB 00:00 2023-09-22T12:42:21.108 INFO:teuthology.orchestra.run.smithi191.stdout:(30/32): python3-zc-lockfile-2.0-2.el8.noarch.r 306 kB/s | 23 kB 00:00 2023-09-22T12:42:23.325 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T12:42:23.543 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T12:42:23.828 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T12:42:23.829 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 1/6 2023-09-22T12:42:23.829 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 2/6 2023-09-22T12:42:23.829 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : jq-1.6-7.el8.x86_64 3/6 2023-09-22T12:42:23.829 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-09-22T12:42:23.829 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-09-22T12:42:24.476 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-09-22T12:42:24.476 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:42:24.476 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T12:42:24.476 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 jq-1.6-7.el8.x86_64 2023-09-22T12:42:24.477 INFO:teuthology.orchestra.run.smithi170.stdout: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 oniguruma-6.8.2-2.el8.x86_64 2023-09-22T12:42:24.477 INFO:teuthology.orchestra.run.smithi170.stdout: socat-1.7.4.1-1.el8.x86_64 xmlstarlet-1.6.1-20.el8.x86_64 2023-09-22T12:42:24.477 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:42:24.477 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T12:42:24.718 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install ceph 2023-09-22T12:42:25.174 INFO:teuthology.orchestra.run.smithi191.stdout:(31/32): python3-mako-1.0.6-14.el8.noarch.rpm 17 kB/s | 157 kB 00:09 2023-09-22T12:42:25.577 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:01:37 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:42:27.150 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:42:27.153 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:42:27.153 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repository Size 2023-09-22T12:42:27.153 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:42:27.153 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T12:42:27.154 INFO:teuthology.orchestra.run.smithi170.stdout: ceph x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.4 k 2023-09-22T12:42:27.154 INFO:teuthology.orchestra.run.smithi170.stdout:Installing dependencies: 2023-09-22T12:42:27.154 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mds x86_64 2:16.2.14-5.g31c126b7.el8 ceph 2.4 M 2023-09-22T12:42:27.154 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr x86_64 2:16.2.14-5.g31c126b7.el8 ceph 1.6 M 2023-09-22T12:42:27.154 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr-modules-core noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 244 k 2023-09-22T12:42:27.154 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mon x86_64 2:16.2.14-5.g31c126b7.el8 ceph 5.0 M 2023-09-22T12:42:27.154 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-osd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 18 M 2023-09-22T12:42:27.154 INFO:teuthology.orchestra.run.smithi170.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-22T12:42:27.155 INFO:teuthology.orchestra.run.smithi170.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-22T12:42:27.155 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-09-22T12:42:27.155 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-22T12:42:27.155 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-22T12:42:27.155 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-09-22T12:42:27.155 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-22T12:42:27.155 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-22T12:42:27.155 INFO:teuthology.orchestra.run.smithi170.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-22T12:42:27.155 INFO:teuthology.orchestra.run.smithi170.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-09-22T12:42:27.156 INFO:teuthology.orchestra.run.smithi170.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-09-22T12:42:27.156 INFO:teuthology.orchestra.run.smithi170.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-22T12:42:27.156 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-22T12:42:27.156 INFO:teuthology.orchestra.run.smithi170.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-22T12:42:27.156 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-09-22T12:42:27.156 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-09-22T12:42:27.156 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-09-22T12:42:27.156 INFO:teuthology.orchestra.run.smithi170.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-22T12:42:27.157 INFO:teuthology.orchestra.run.smithi170.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-22T12:42:27.157 INFO:teuthology.orchestra.run.smithi170.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-22T12:42:27.157 INFO:teuthology.orchestra.run.smithi170.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-22T12:42:27.157 INFO:teuthology.orchestra.run.smithi170.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-22T12:42:27.157 INFO:teuthology.orchestra.run.smithi170.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-22T12:42:27.157 INFO:teuthology.orchestra.run.smithi170.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-22T12:42:27.157 INFO:teuthology.orchestra.run.smithi170.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-09-22T12:42:27.157 INFO:teuthology.orchestra.run.smithi170.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-22T12:42:27.158 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:42:27.158 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T12:42:27.158 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:42:27.158 INFO:teuthology.orchestra.run.smithi170.stdout:Install 32 Packages 2023-09-22T12:42:27.158 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:42:27.159 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 31 M 2023-09-22T12:42:27.159 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 101 M 2023-09-22T12:42:27.159 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T12:42:27.932 INFO:teuthology.orchestra.run.smithi170.stdout:(1/32): ceph-16.2.14-5.g31c126b7.el8.x86_64.rpm 27 kB/s | 6.4 kB 00:00 2023-09-22T12:42:28.023 INFO:teuthology.orchestra.run.smithi170.stdout:(2/32): ceph-mds-16.2.14-5.g31c126b7.el8.x86_64 7.4 MB/s | 2.4 MB 00:00 2023-09-22T12:42:28.057 INFO:teuthology.orchestra.run.smithi170.stdout:(3/32): ceph-mgr-16.2.14-5.g31c126b7.el8.x86_64 4.4 MB/s | 1.6 MB 00:00 2023-09-22T12:42:28.157 INFO:teuthology.orchestra.run.smithi170.stdout:(4/32): ceph-mgr-modules-core-16.2.14-5.g31c126 2.4 MB/s | 244 kB 00:00 2023-09-22T12:42:28.891 INFO:teuthology.orchestra.run.smithi170.stdout:(5/32): ceph-osd-16.2.14-5.g31c126b7.el8.x86_64 21 MB/s | 18 MB 00:00 2023-09-22T12:42:29.258 INFO:teuthology.orchestra.run.smithi170.stdout:(6/32): python3-markupsafe-0.23-19.el8.x86_64.r 106 kB/s | 39 kB 00:00 2023-09-22T12:42:29.508 INFO:teuthology.orchestra.run.smithi170.stdout:(7/32): ceph-mon-16.2.14-5.g31c126b7.el8.x86_64 3.2 MB/s | 5.0 MB 00:01 2023-09-22T12:42:33.874 INFO:teuthology.orchestra.run.smithi170.stdout:(8/32): python3-pytz-2017.2-11.el8.noarch.rpm 13 kB/s | 57 kB 00:04 2023-09-22T12:42:33.900 INFO:teuthology.orchestra.run.smithi170.stdout:(9/32): python3-mako-1.0.6-14.el8.noarch.rpm 27 kB/s | 157 kB 00:05 2023-09-22T12:42:34.090 INFO:teuthology.orchestra.run.smithi191.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 29 kB/s | 457 kB 00:15 2023-09-22T12:42:34.092 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:42:34.093 INFO:teuthology.orchestra.run.smithi191.stdout:Total 1.6 MB/s | 31 MB 00:19 2023-09-22T12:42:34.176 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T12:42:34.215 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T12:42:34.215 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T12:42:34.525 INFO:teuthology.orchestra.run.smithi170.stdout:(10/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 398 kB/s | 249 kB 00:00 2023-09-22T12:42:34.539 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T12:42:34.543 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T12:42:35.367 INFO:teuthology.orchestra.run.smithi170.stdout:(11/32): python3-cryptography-3.2.1-6.el8.x86_6 762 kB/s | 641 kB 00:00 2023-09-22T12:42:35.525 INFO:teuthology.orchestra.run.smithi170.stdout:(12/32): python3-pycparser-2.14-14.el8.noarch.r 689 kB/s | 109 kB 00:00 2023-09-22T12:42:35.670 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T12:42:35.946 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-22T12:42:36.000 INFO:teuthology.orchestra.run.smithi170.stdout:(13/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 92 kB/s | 44 kB 00:00 2023-09-22T12:42:36.189 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-22T12:42:36.359 INFO:teuthology.orchestra.run.smithi170.stdout:(14/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 517 kB/s | 185 kB 00:00 2023-09-22T12:42:36.407 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-09-22T12:42:36.543 INFO:teuthology.orchestra.run.smithi170.stdout:(15/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 941 kB/s | 173 kB 00:00 2023-09-22T12:42:36.696 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-09-22T12:42:36.793 INFO:teuthology.orchestra.run.smithi170.stdout:(16/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.5 MB/s | 384 kB 00:00 2023-09-22T12:42:36.877 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-09-22T12:42:36.918 INFO:teuthology.orchestra.run.smithi170.stdout:(17/32): python3-jaraco-6.2-6.el8.noarch.rpm 89 kB/s | 11 kB 00:00 2023-09-22T12:42:37.032 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-09-22T12:42:37.035 INFO:teuthology.orchestra.run.smithi170.stdout:(18/32): python3-jaraco-functools-2.0-4.el8.noa 151 kB/s | 18 kB 00:00 2023-09-22T12:42:37.169 INFO:teuthology.orchestra.run.smithi170.stdout:(19/32): python3-logutils-0.3.5-11.el8.noarch.r 365 kB/s | 49 kB 00:00 2023-09-22T12:42:37.175 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-09-22T12:42:37.294 INFO:teuthology.orchestra.run.smithi170.stdout:(20/32): python3-more-itertools-7.2.0-3.el8.noa 470 kB/s | 59 kB 00:00 2023-09-22T12:42:37.395 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-09-22T12:42:37.444 INFO:teuthology.orchestra.run.smithi170.stdout:(21/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.8 MB/s | 283 kB 00:00 2023-09-22T12:42:37.561 INFO:teuthology.orchestra.run.smithi170.stdout:(22/32): python3-portend-2.6-1.el8.noarch.rpm 133 kB/s | 16 kB 00:00 2023-09-22T12:42:37.596 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-09-22T12:42:37.686 INFO:teuthology.orchestra.run.smithi170.stdout:(23/32): python3-simplegeneric-0.8.1-17.el8.noa 150 kB/s | 19 kB 00:00 2023-09-22T12:42:37.824 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-09-22T12:42:37.828 INFO:teuthology.orchestra.run.smithi170.stdout:(24/32): python3-singledispatch-3.4.0.3-18.el8. 169 kB/s | 24 kB 00:00 2023-09-22T12:42:37.953 INFO:teuthology.orchestra.run.smithi170.stdout:(25/32): python3-tempora-1.14.1-5.el8.noarch.rp 234 kB/s | 29 kB 00:00 2023-09-22T12:42:38.038 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-09-22T12:42:38.079 INFO:teuthology.orchestra.run.smithi170.stdout:(26/32): python3-trustme-0.6.0-4.el8.noarch.rpm 220 kB/s | 27 kB 00:00 2023-09-22T12:42:38.229 INFO:teuthology.orchestra.run.smithi170.stdout:(27/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.7 MB/s | 255 kB 00:00 2023-09-22T12:42:38.262 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-09-22T12:42:38.388 INFO:teuthology.orchestra.run.smithi170.stdout:(28/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.5 MB/s | 251 kB 00:00 2023-09-22T12:42:38.513 INFO:teuthology.orchestra.run.smithi170.stdout:(29/32): python3-webtest-2.0.33-1.el8.noarch.rp 683 kB/s | 85 kB 00:00 2023-09-22T12:42:38.637 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-09-22T12:42:38.638 INFO:teuthology.orchestra.run.smithi170.stdout:(30/32): python3-zc-lockfile-2.0-2.el8.noarch.r 184 kB/s | 23 kB 00:00 2023-09-22T12:42:38.942 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-09-22T12:42:39.151 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-09-22T12:42:39.347 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-09-22T12:42:39.590 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T12:42:39.882 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-09-22T12:42:40.136 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-09-22T12:42:40.324 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-09-22T12:42:40.510 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-22T12:42:40.756 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-22T12:42:42.146 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-22T12:42:43.307 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-09-22T12:42:43.381 INFO:teuthology.orchestra.run.smithi170.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 7.3 kB/s | 103 kB 00:14 2023-09-22T12:42:43.604 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-09-22T12:42:43.918 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-22T12:42:44.315 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 27/32 2023-09-22T12:42:44.489 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T12:42:45.986 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T12:42:46.163 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T12:42:46.588 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T12:42:46.771 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T12:42:46.988 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T12:42:47.200 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T12:42:47.218 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T12:42:47.337 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T12:42:47.747 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T12:42:47.748 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 1/32 2023-09-22T12:42:47.748 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2/32 2023-09-22T12:42:47.748 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 3/32 2023-09-22T12:42:47.748 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 4/32 2023-09-22T12:42:47.748 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 5/32 2023-09-22T12:42:47.748 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 6/32 2023-09-22T12:42:47.748 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-09-22T12:42:47.748 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-09-22T12:42:47.748 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-09-22T12:42:47.748 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-09-22T12:42:47.749 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-09-22T12:42:47.750 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-09-22T12:42:47.750 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-09-22T12:42:47.750 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-09-22T12:42:47.750 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-09-22T12:42:47.750 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-09-22T12:42:47.750 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-09-22T12:42:47.750 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-09-22T12:42:47.750 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-09-22T12:42:47.750 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-09-22T12:42:48.937 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-09-22T12:42:48.937 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:48.937 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T12:42:48.937 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:42:48.937 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:42:48.937 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:42:48.937 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:42:48.938 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:42:48.938 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:42:48.938 INFO:teuthology.orchestra.run.smithi191.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-22T12:42:48.938 INFO:teuthology.orchestra.run.smithi191.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-22T12:42:48.938 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-09-22T12:42:48.938 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-22T12:42:48.938 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-22T12:42:48.938 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-09-22T12:42:48.938 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-22T12:42:48.938 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-22T12:42:48.938 INFO:teuthology.orchestra.run.smithi191.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-mako-1.0.6-14.el8.noarch 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-portend-2.6-1.el8.noarch 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pytz-2017.2-11.el8.noarch 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-22T12:42:48.939 INFO:teuthology.orchestra.run.smithi191.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-22T12:42:48.940 INFO:teuthology.orchestra.run.smithi191.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-22T12:42:48.940 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-22T12:42:48.940 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-22T12:42:48.940 INFO:teuthology.orchestra.run.smithi191.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-22T12:42:48.940 INFO:teuthology.orchestra.run.smithi191.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-22T12:42:48.940 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:48.940 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T12:42:49.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-base 2023-09-22T12:42:49.996 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:01 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:42:50.262 INFO:teuthology.orchestra.run.smithi170.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 28 kB/s | 457 kB 00:16 2023-09-22T12:42:50.266 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:42:50.266 INFO:teuthology.orchestra.run.smithi170.stdout:Total 1.4 MB/s | 31 MB 00:23 2023-09-22T12:42:50.349 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T12:42:50.388 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T12:42:50.388 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T12:42:50.709 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T12:42:50.712 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T12:42:51.397 INFO:teuthology.orchestra.run.smithi191.stdout:Package ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T12:42:51.577 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T12:42:51.578 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-09-22T12:42:51.578 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T12:42:51.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install cephadm 2023-09-22T12:42:51.783 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T12:42:51.987 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-22T12:42:52.197 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-22T12:42:52.398 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-09-22T12:42:52.540 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:04 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:42:52.671 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-09-22T12:42:52.877 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-09-22T12:42:53.066 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-09-22T12:42:53.251 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-09-22T12:42:53.479 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-09-22T12:42:53.830 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-09-22T12:42:54.122 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T12:42:54.123 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:42:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-09-22T12:42:54.123 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:42:54.123 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T12:42:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 84 k 2023-09-22T12:42:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:54.123 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T12:42:54.124 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:42:54.124 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-09-22T12:42:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:54.124 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 84 k 2023-09-22T12:42:54.124 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 348 k 2023-09-22T12:42:54.124 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T12:42:54.294 INFO:teuthology.orchestra.run.smithi191.stdout:cephadm-16.2.14-5.g31c126b7.el8.noarch.rpm 496 kB/s | 84 kB 00:00 2023-09-22T12:42:54.295 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:42:54.295 INFO:teuthology.orchestra.run.smithi191.stdout:Total 492 kB/s | 84 kB 00:00 2023-09-22T12:42:54.295 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T12:42:54.302 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T12:42:54.302 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T12:42:54.304 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T12:42:54.307 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T12:42:54.351 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T12:42:54.892 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:42:55.049 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:42:55.275 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:42:55.625 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-09-22T12:42:55.838 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-09-22T12:42:55.877 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:42:55.877 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:55.877 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T12:42:55.877 INFO:teuthology.orchestra.run.smithi191.stdout: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:42:55.877 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:55.877 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T12:42:56.045 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-09-22T12:42:56.083 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-immutable-object-cache 2023-09-22T12:42:56.420 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-09-22T12:42:56.684 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-09-22T12:42:56.892 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-09-22T12:42:56.945 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:08 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:42:57.122 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-09-22T12:42:57.365 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T12:42:57.691 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-09-22T12:42:57.886 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-09-22T12:42:58.091 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-09-22T12:42:58.277 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-22T12:42:58.514 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T12:42:58.515 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:42:58.515 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repo Size 2023-09-22T12:42:58.515 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:42:58.515 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T12:42:58.515 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-5.g31c126b7.el8 ceph 161 k 2023-09-22T12:42:58.515 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:58.516 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T12:42:58.516 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:42:58.516 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-09-22T12:42:58.516 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:42:58.516 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 161 k 2023-09-22T12:42:58.516 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 470 k 2023-09-22T12:42:58.517 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T12:42:58.537 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-22T12:42:58.761 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-22T12:42:58.974 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-09-22T12:42:58.982 INFO:teuthology.orchestra.run.smithi191.stdout:ceph-immutable-object-cache-16.2.14-5.g31c126b7 346 kB/s | 161 kB 00:00 2023-09-22T12:42:58.982 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:42:58.983 INFO:teuthology.orchestra.run.smithi191.stdout:Total 345 kB/s | 161 kB 00:00 2023-09-22T12:42:58.983 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T12:42:58.994 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T12:42:58.994 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T12:42:59.041 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T12:42:59.044 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T12:42:59.153 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T12:42:59.302 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-09-22T12:42:59.319 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:42:59.602 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-22T12:42:59.740 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:42:59.953 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 27/32 2023-09-22T12:43:00.117 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T12:43:00.346 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:43:00.346 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:43:00.346 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T12:43:00.347 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:43:00.347 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:43:00.347 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T12:43:00.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr 2023-09-22T12:43:01.408 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:13 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:43:01.619 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T12:43:01.772 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T12:43:02.199 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T12:43:02.372 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T12:43:02.588 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T12:43:02.775 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T12:43:02.788 INFO:teuthology.orchestra.run.smithi191.stdout:Package ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T12:43:02.793 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T12:43:02.904 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T12:43:02.968 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T12:43:02.969 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-09-22T12:43:02.969 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T12:43:03.085 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-dashboard 2023-09-22T12:43:03.310 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T12:43:03.310 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 1/32 2023-09-22T12:43:03.310 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2/32 2023-09-22T12:43:03.311 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 3/32 2023-09-22T12:43:03.311 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 4/32 2023-09-22T12:43:03.311 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 5/32 2023-09-22T12:43:03.311 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 6/32 2023-09-22T12:43:03.311 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-09-22T12:43:03.311 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-09-22T12:43:03.311 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-09-22T12:43:03.311 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-09-22T12:43:03.311 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-09-22T12:43:03.311 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-09-22T12:43:03.311 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-09-22T12:43:03.312 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-09-22T12:43:03.312 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-09-22T12:43:03.312 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-09-22T12:43:03.312 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T12:43:03.312 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-09-22T12:43:03.312 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-09-22T12:43:03.312 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-09-22T12:43:03.312 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-09-22T12:43:03.312 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-09-22T12:43:03.312 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-09-22T12:43:03.312 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-09-22T12:43:03.313 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-09-22T12:43:03.313 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-09-22T12:43:03.313 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-09-22T12:43:03.314 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-09-22T12:43:03.314 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-09-22T12:43:03.314 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-09-22T12:43:03.314 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-09-22T12:43:03.942 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:15 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:43:04.348 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-09-22T12:43:04.348 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:04.348 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T12:43:04.348 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:43:04.348 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:43:04.348 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:43:04.348 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-22T12:43:04.349 INFO:teuthology.orchestra.run.smithi170.stdout: python3-mako-1.0.6-14.el8.noarch 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-portend-2.6-1.el8.noarch 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pytz-2017.2-11.el8.noarch 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-22T12:43:04.350 INFO:teuthology.orchestra.run.smithi170.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-22T12:43:04.351 INFO:teuthology.orchestra.run.smithi170.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-22T12:43:04.351 INFO:teuthology.orchestra.run.smithi170.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-22T12:43:04.351 INFO:teuthology.orchestra.run.smithi170.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-22T12:43:04.351 INFO:teuthology.orchestra.run.smithi170.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-22T12:43:04.351 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:04.351 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T12:43:04.480 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install ceph-base 2023-09-22T12:43:05.334 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:02:17 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:43:05.537 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T12:43:05.538 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:43:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-09-22T12:43:05.539 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:43:05.539 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T12:43:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-dashboard noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 3.1 M 2023-09-22T12:43:05.539 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-09-22T12:43:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-grafana-dashboards noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 24 k 2023-09-22T12:43:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-prometheus-alerts noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 15 k 2023-09-22T12:43:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-09-22T12:43:05.539 INFO:teuthology.orchestra.run.smithi191.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-22T12:43:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-22T12:43:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:43:05.540 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T12:43:05.540 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:43:05.540 INFO:teuthology.orchestra.run.smithi191.stdout:Install 6 Packages 2023-09-22T12:43:05.540 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:43:05.540 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 3.4 M 2023-09-22T12:43:05.541 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 22 M 2023-09-22T12:43:05.541 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T12:43:05.834 INFO:teuthology.orchestra.run.smithi191.stdout:(1/6): ceph-grafana-dashboards-16.2.14-5.g31c12 148 kB/s | 24 kB 00:00 2023-09-22T12:43:05.859 INFO:teuthology.orchestra.run.smithi191.stdout:(2/6): ceph-prometheus-alerts-16.2.14-5.g31c126 78 kB/s | 15 kB 00:00 2023-09-22T12:43:06.060 INFO:teuthology.orchestra.run.smithi191.stdout:(3/6): ceph-mgr-dashboard-16.2.14-5.g31c126b7.e 8.0 MB/s | 3.1 MB 00:00 2023-09-22T12:43:06.218 INFO:teuthology.orchestra.run.smithi191.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 111 kB/s | 43 kB 00:00 2023-09-22T12:43:06.302 INFO:teuthology.orchestra.run.smithi191.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 76 kB/s | 34 kB 00:00 2023-09-22T12:43:06.728 INFO:teuthology.orchestra.run.smithi170.stdout:Package ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T12:43:06.743 INFO:teuthology.orchestra.run.smithi191.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 288 kB/s | 196 kB 00:00 2023-09-22T12:43:06.746 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:43:06.747 INFO:teuthology.orchestra.run.smithi191.stdout:Total 2.8 MB/s | 3.4 MB 00:01 2023-09-22T12:43:06.758 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T12:43:06.768 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T12:43:06.768 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T12:43:06.908 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:43:06.909 INFO:teuthology.orchestra.run.smithi170.stdout:Nothing to do. 2023-09-22T12:43:06.909 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T12:43:06.940 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T12:43:06.944 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T12:43:07.018 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install cephadm 2023-09-22T12:43:07.250 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T12:43:07.440 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-09-22T12:43:07.659 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-09-22T12:43:07.844 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-09-22T12:43:07.869 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:02:19 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:43:07.967 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 4/6 2023-09-22T12:43:08.976 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 5/6 2023-09-22T12:43:09.451 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:43:09.452 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:43:09.452 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repository Size 2023-09-22T12:43:09.452 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:43:09.452 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T12:43:09.453 INFO:teuthology.orchestra.run.smithi170.stdout: cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 84 k 2023-09-22T12:43:09.453 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:09.453 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T12:43:09.453 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:43:09.453 INFO:teuthology.orchestra.run.smithi170.stdout:Install 1 Package 2023-09-22T12:43:09.453 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:09.453 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 84 k 2023-09-22T12:43:09.453 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 348 k 2023-09-22T12:43:09.454 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T12:43:09.601 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T12:43:09.612 INFO:teuthology.orchestra.run.smithi170.stdout:cephadm-16.2.14-5.g31c126b7.el8.noarch.rpm 530 kB/s | 84 kB 00:00 2023-09-22T12:43:09.612 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:43:09.613 INFO:teuthology.orchestra.run.smithi170.stdout:Total 525 kB/s | 84 kB 00:00 2023-09-22T12:43:09.613 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T12:43:09.620 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T12:43:09.620 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T12:43:09.622 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T12:43:09.625 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T12:43:09.672 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T12:43:09.906 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T12:43:09.906 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 1/6 2023-09-22T12:43:09.906 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2/6 2023-09-22T12:43:09.906 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 3/6 2023-09-22T12:43:09.906 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-09-22T12:43:09.906 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-09-22T12:43:10.177 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:43:10.337 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:43:10.536 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-09-22T12:43:10.536 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:43:10.536 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T12:43:10.536 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:43:10.536 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:43:10.536 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:43:10.536 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-22T12:43:10.536 INFO:teuthology.orchestra.run.smithi191.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-22T12:43:10.536 INFO:teuthology.orchestra.run.smithi191.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-22T12:43:10.536 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:43:10.536 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T12:43:10.559 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:43:10.667 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-22T12:43:11.166 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:43:11.166 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:11.167 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T12:43:11.167 INFO:teuthology.orchestra.run.smithi170.stdout: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:43:11.167 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:11.167 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T12:43:11.401 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install ceph-immutable-object-cache 2023-09-22T12:43:11.519 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:02:23 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:43:12.254 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:02:24 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:43:13.099 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T12:43:13.100 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================================ 2023-09-22T12:43:13.100 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-09-22T12:43:13.100 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================================ 2023-09-22T12:43:13.100 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T12:43:13.101 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 8.6 M 2023-09-22T12:43:13.101 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-09-22T12:43:13.101 INFO:teuthology.orchestra.run.smithi191.stdout: libgfortran x86_64 8.5.0-20.el8 baseos 645 k 2023-09-22T12:43:13.101 INFO:teuthology.orchestra.run.smithi191.stdout: libquadmath x86_64 8.5.0-20.el8 baseos 172 k 2023-09-22T12:43:13.101 INFO:teuthology.orchestra.run.smithi191.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-09-22T12:43:13.101 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-09-22T12:43:13.101 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-09-22T12:43:13.101 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-09-22T12:43:13.101 INFO:teuthology.orchestra.run.smithi191.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-09-22T12:43:13.101 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:43:13.101 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T12:43:13.102 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================================ 2023-09-22T12:43:13.102 INFO:teuthology.orchestra.run.smithi191.stdout:Install 8 Packages 2023-09-22T12:43:13.102 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:43:13.102 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 37 M 2023-09-22T12:43:13.102 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 210 M 2023-09-22T12:43:13.102 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T12:43:13.790 INFO:teuthology.orchestra.run.smithi191.stdout:(1/8): ceph-mgr-diskprediction-local-16.2.14-5. 14 MB/s | 8.6 MB 00:00 2023-09-22T12:43:13.832 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:43:13.833 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:43:13.833 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repo Size 2023-09-22T12:43:13.833 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:43:13.833 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T12:43:13.833 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-5.g31c126b7.el8 ceph 161 k 2023-09-22T12:43:13.833 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:13.833 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T12:43:13.833 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:43:13.833 INFO:teuthology.orchestra.run.smithi170.stdout:Install 1 Package 2023-09-22T12:43:13.834 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:13.834 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 161 k 2023-09-22T12:43:13.834 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 470 k 2023-09-22T12:43:13.834 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T12:43:14.025 INFO:teuthology.orchestra.run.smithi170.stdout:ceph-immutable-object-cache-16.2.14-5.g31c126b7 846 kB/s | 161 kB 00:00 2023-09-22T12:43:14.025 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:43:14.026 INFO:teuthology.orchestra.run.smithi170.stdout:Total 840 kB/s | 161 kB 00:00 2023-09-22T12:43:14.026 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T12:43:14.036 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T12:43:14.036 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T12:43:14.082 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T12:43:14.085 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T12:43:14.192 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T12:43:14.345 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:43:14.759 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:43:15.392 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:43:15.392 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:15.392 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T12:43:15.392 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:43:15.393 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:15.393 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T12:43:15.589 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install ceph-mgr 2023-09-22T12:43:16.463 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:02:28 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:43:17.859 INFO:teuthology.orchestra.run.smithi170.stdout:Package ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T12:43:18.041 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:43:18.042 INFO:teuthology.orchestra.run.smithi170.stdout:Nothing to do. 2023-09-22T12:43:18.042 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T12:43:18.149 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install ceph-mgr-dashboard 2023-09-22T12:43:19.009 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:02:31 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:43:20.584 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:43:20.585 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:43:20.585 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repository Size 2023-09-22T12:43:20.585 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:43:20.585 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T12:43:20.585 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr-dashboard noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 3.1 M 2023-09-22T12:43:20.585 INFO:teuthology.orchestra.run.smithi170.stdout:Installing dependencies: 2023-09-22T12:43:20.585 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-grafana-dashboards noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 24 k 2023-09-22T12:43:20.586 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-prometheus-alerts noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 15 k 2023-09-22T12:43:20.586 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-09-22T12:43:20.586 INFO:teuthology.orchestra.run.smithi170.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-22T12:43:20.586 INFO:teuthology.orchestra.run.smithi170.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-22T12:43:20.586 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:20.586 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T12:43:20.586 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:43:20.586 INFO:teuthology.orchestra.run.smithi170.stdout:Install 6 Packages 2023-09-22T12:43:20.586 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:20.587 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 3.4 M 2023-09-22T12:43:20.587 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 22 M 2023-09-22T12:43:20.587 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T12:43:21.620 INFO:teuthology.orchestra.run.smithi170.stdout:(1/6): ceph-prometheus-alerts-16.2.14-5.g31c126 30 kB/s | 15 kB 00:00 2023-09-22T12:43:21.645 INFO:teuthology.orchestra.run.smithi170.stdout:(2/6): ceph-grafana-dashboards-16.2.14-5.g31c12 47 kB/s | 24 kB 00:00 2023-09-22T12:43:21.721 INFO:teuthology.orchestra.run.smithi170.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 449 kB/s | 34 kB 00:00 2023-09-22T12:43:21.771 INFO:teuthology.orchestra.run.smithi170.stdout:(4/6): python3-routes-2.4.1-12.el8.noarch.rpm 3.8 MB/s | 196 kB 00:00 2023-09-22T12:43:22.046 INFO:teuthology.orchestra.run.smithi170.stdout:(5/6): ceph-mgr-dashboard-16.2.14-5.g31c126b7.e 3.4 MB/s | 3.1 MB 00:00 2023-09-22T12:43:22.071 INFO:teuthology.orchestra.run.smithi170.stdout:(6/6): python3-jwt-1.6.1-2.el8.noarch.rpm 94 kB/s | 43 kB 00:00 2023-09-22T12:43:22.073 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:43:22.074 INFO:teuthology.orchestra.run.smithi170.stdout:Total 2.3 MB/s | 3.4 MB 00:01 2023-09-22T12:43:22.086 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T12:43:22.097 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T12:43:22.097 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T12:43:22.264 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T12:43:22.267 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T12:43:22.542 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T12:43:22.716 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-09-22T12:43:22.936 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-09-22T12:43:23.112 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-09-22T12:43:23.241 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 4/6 2023-09-22T12:43:24.247 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 5/6 2023-09-22T12:43:24.480 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T12:43:24.774 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T12:43:24.774 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 1/6 2023-09-22T12:43:24.774 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2/6 2023-09-22T12:43:24.774 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 3/6 2023-09-22T12:43:24.774 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-09-22T12:43:24.775 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-09-22T12:43:25.392 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-09-22T12:43:25.392 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:25.392 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T12:43:25.392 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:43:25.392 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:43:25.392 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:43:25.393 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-22T12:43:25.393 INFO:teuthology.orchestra.run.smithi170.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-22T12:43:25.393 INFO:teuthology.orchestra.run.smithi170.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-22T12:43:25.393 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:25.393 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T12:43:25.529 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-22T12:43:26.381 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:02:38 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:43:27.956 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:43:27.958 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================================ 2023-09-22T12:43:27.958 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repository Size 2023-09-22T12:43:27.958 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================================ 2023-09-22T12:43:27.958 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T12:43:27.958 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 8.6 M 2023-09-22T12:43:27.958 INFO:teuthology.orchestra.run.smithi170.stdout:Installing dependencies: 2023-09-22T12:43:27.959 INFO:teuthology.orchestra.run.smithi170.stdout: libgfortran x86_64 8.5.0-20.el8 baseos 645 k 2023-09-22T12:43:27.959 INFO:teuthology.orchestra.run.smithi170.stdout: libquadmath x86_64 8.5.0-20.el8 baseos 172 k 2023-09-22T12:43:27.959 INFO:teuthology.orchestra.run.smithi170.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-09-22T12:43:27.959 INFO:teuthology.orchestra.run.smithi170.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-09-22T12:43:27.959 INFO:teuthology.orchestra.run.smithi170.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-09-22T12:43:27.959 INFO:teuthology.orchestra.run.smithi170.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-09-22T12:43:27.959 INFO:teuthology.orchestra.run.smithi170.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-09-22T12:43:27.959 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:27.960 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T12:43:27.960 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================================ 2023-09-22T12:43:27.960 INFO:teuthology.orchestra.run.smithi170.stdout:Install 8 Packages 2023-09-22T12:43:27.960 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:43:27.961 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 37 M 2023-09-22T12:43:27.961 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 210 M 2023-09-22T12:43:27.961 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T12:43:28.934 INFO:teuthology.orchestra.run.smithi170.stdout:(1/8): ceph-mgr-diskprediction-local-16.2.14-5. 14 MB/s | 8.6 MB 00:00 2023-09-22T12:45:54.071 INFO:teuthology.orchestra.run.smithi191.stdout:(2/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 24 kB/s | 3.7 MB 02:40 2023-09-22T12:45:58.810 INFO:teuthology.orchestra.run.smithi170.stdout:(2/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 25 kB/s | 3.7 MB 02:29 2023-09-22T12:46:00.635 INFO:teuthology.orchestra.run.smithi191.stdout:(3/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 34 kB/s | 225 kB 00:06 2023-09-22T12:46:17.015 INFO:teuthology.orchestra.run.smithi170.stdout:(3/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 12 kB/s | 225 kB 00:18 2023-09-22T12:46:36.784 INFO:teuthology.orchestra.run.smithi191.stdout:(4/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 24 kB/s | 4.9 MB 03:23 2023-09-22T12:46:37.422 INFO:teuthology.orchestra.run.smithi191.stdout:(5/8): libgfortran-8.5.0-20.el8.x86_64.rpm 1.0 MB/s | 645 kB 00:00 2023-09-22T12:46:37.530 INFO:teuthology.orchestra.run.smithi191.stdout:(6/8): libquadmath-8.5.0-20.el8.x86_64.rpm 1.6 MB/s | 172 kB 00:00 2023-09-22T12:47:06.851 INFO:teuthology.orchestra.run.smithi170.stdout:(4/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 23 kB/s | 4.9 MB 03:38 2023-09-22T12:47:07.376 INFO:teuthology.orchestra.run.smithi170.stdout:(5/8): libgfortran-8.5.0-20.el8.x86_64.rpm 1.2 MB/s | 645 kB 00:00 2023-09-22T12:47:07.476 INFO:teuthology.orchestra.run.smithi170.stdout:(6/8): libquadmath-8.5.0-20.el8.x86_64.rpm 1.7 MB/s | 172 kB 00:00 2023-09-22T12:47:18.436 INFO:teuthology.orchestra.run.smithi191.stdout:(7/8): openblas-0.3.15-4.el8.x86_64.rpm 20 kB/s | 4.7 MB 04:05 2023-09-22T12:47:20.174 INFO:teuthology.orchestra.run.smithi170.stdout:(7/8): openblas-0.3.15-4.el8.x86_64.rpm 21 kB/s | 4.7 MB 03:51 2023-09-22T12:56:33.986 INFO:teuthology.orchestra.run.smithi170.stdout:(8/8): python3-scipy-1.0.0-21.module_el8.5.0+77 23 kB/s | 14 MB 10:16 2023-09-22T12:56:33.987 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:56:33.987 INFO:teuthology.orchestra.run.smithi170.stdout:Total 48 kB/s | 37 MB 13:06 2023-09-22T12:56:34.213 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T12:56:34.231 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T12:56:34.232 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T12:56:34.552 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T12:56:34.555 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T12:56:35.112 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T12:56:35.264 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T12:56:35.395 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T12:56:35.522 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T12:56:36.115 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T12:56:36.297 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T12:56:36.913 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T12:56:37.047 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T12:56:37.753 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T12:56:38.074 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T12:56:38.338 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T12:56:40.090 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T12:56:41.385 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-09-22T12:56:41.587 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T12:56:41.901 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T12:56:41.901 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 1/8 2023-09-22T12:56:41.901 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-09-22T12:56:41.901 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-09-22T12:56:41.901 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-09-22T12:56:41.901 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T12:56:41.902 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-09-22T12:56:41.902 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 7/8 2023-09-22T12:56:42.525 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 8/8 2023-09-22T12:56:42.525 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:56:42.525 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T12:56:42.526 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:56:42.526 INFO:teuthology.orchestra.run.smithi170.stdout: libgfortran-8.5.0-20.el8.x86_64 2023-09-22T12:56:42.526 INFO:teuthology.orchestra.run.smithi170.stdout: libquadmath-8.5.0-20.el8.x86_64 2023-09-22T12:56:42.526 INFO:teuthology.orchestra.run.smithi170.stdout: openblas-0.3.15-4.el8.x86_64 2023-09-22T12:56:42.526 INFO:teuthology.orchestra.run.smithi170.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-09-22T12:56:42.526 INFO:teuthology.orchestra.run.smithi170.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-22T12:56:42.526 INFO:teuthology.orchestra.run.smithi170.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-22T12:56:42.526 INFO:teuthology.orchestra.run.smithi170.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-09-22T12:56:42.527 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:56:42.527 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T12:56:42.647 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install ceph-mgr-rook 2023-09-22T12:56:43.507 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:15:55 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:56:45.067 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:56:45.069 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:56:45.069 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repository Size 2023-09-22T12:56:45.069 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:56:45.069 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T12:56:45.069 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr-rook noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 27 k 2023-09-22T12:56:45.069 INFO:teuthology.orchestra.run.smithi170.stdout:Installing dependencies: 2023-09-22T12:56:45.069 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-22T12:56:45.070 INFO:teuthology.orchestra.run.smithi170.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-22T12:56:45.070 INFO:teuthology.orchestra.run.smithi170.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-22T12:56:45.070 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-09-22T12:56:45.070 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-09-22T12:56:45.070 INFO:teuthology.orchestra.run.smithi170.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-22T12:56:45.070 INFO:teuthology.orchestra.run.smithi170.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-09-22T12:56:45.070 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-09-22T12:56:45.070 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-09-22T12:56:45.070 INFO:teuthology.orchestra.run.smithi170.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-09-22T12:56:45.070 INFO:teuthology.orchestra.run.smithi170.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-22T12:56:45.070 INFO:teuthology.orchestra.run.smithi170.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-22T12:56:45.071 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:56:45.071 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T12:56:45.071 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:56:45.071 INFO:teuthology.orchestra.run.smithi170.stdout:Install 13 Packages 2023-09-22T12:56:45.071 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:56:45.071 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 1.9 M 2023-09-22T12:56:45.071 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 24 M 2023-09-22T12:56:45.071 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T12:56:45.908 INFO:teuthology.orchestra.run.smithi170.stdout:(1/13): ceph-mgr-rook-16.2.14-5.g31c126b7.el8.n 41 kB/s | 27 kB 00:00 2023-09-22T12:56:46.750 INFO:teuthology.orchestra.run.smithi170.stdout:(2/13): python3-jsonpointer-1.10-11.el8.noarch. 13 kB/s | 20 kB 00:01 2023-09-22T12:56:47.325 INFO:teuthology.orchestra.run.smithi170.stdout:(3/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 13 kB/s | 27 kB 00:02 2023-09-22T12:56:47.792 INFO:teuthology.orchestra.run.smithi170.stdout:(4/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 332 kB/s | 155 kB 00:00 2023-09-22T12:56:47.891 INFO:teuthology.orchestra.run.smithi170.stdout:(5/13): python3-requests-oauthlib-1.0.0-1.el8.n 427 kB/s | 43 kB 00:00 2023-09-22T12:56:48.392 INFO:teuthology.orchestra.run.smithi170.stdout:(6/13): python3-cachetools-3.1.1-4.el8.noarch.r 66 kB/s | 33 kB 00:00 2023-09-22T12:56:48.484 INFO:teuthology.orchestra.run.smithi170.stdout:(7/13): python3-certifi-2018.10.15-7.el8.noarch 178 kB/s | 16 kB 00:00 2023-09-22T12:56:48.642 INFO:teuthology.orchestra.run.smithi170.stdout:(8/13): python3-google-auth-1.1.1-10.el8.noarch 513 kB/s | 81 kB 00:00 2023-09-22T12:56:48.943 INFO:teuthology.orchestra.run.smithi170.stdout:(9/13): python3-kubernetes-11.0.0-6.el8.noarch. 3.9 MB/s | 1.2 MB 00:00 2023-09-22T12:56:49.018 INFO:teuthology.orchestra.run.smithi170.stdout:(10/13): python3-rsa-4.9-2.el8.noarch.rpm 857 kB/s | 64 kB 00:00 2023-09-22T12:56:49.093 INFO:teuthology.orchestra.run.smithi170.stdout:(11/13): python3-websocket-client-0.56.0-5.el8. 816 kB/s | 61 kB 00:00 2023-09-22T12:56:50.069 INFO:teuthology.orchestra.run.smithi170.stdout:(12/13): python3-pyasn1-modules-0.3.7-6.el8.noa 33 kB/s | 110 kB 00:03 2023-09-22T12:56:55.618 INFO:teuthology.orchestra.run.smithi170.stdout:(13/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 13 kB/s | 126 kB 00:09 2023-09-22T12:56:55.620 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:56:55.620 INFO:teuthology.orchestra.run.smithi170.stdout:Total 185 kB/s | 1.9 MB 00:10 2023-09-22T12:56:55.680 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T12:56:55.696 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T12:56:55.696 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T12:56:55.783 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T12:56:55.786 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T12:56:56.280 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T12:56:56.537 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-22T12:56:56.718 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-09-22T12:56:56.937 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-09-22T12:56:57.150 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-09-22T12:56:57.261 INFO:teuthology.orchestra.run.smithi191.stdout:(8/8): python3-scipy-1.0.0-21.module_el8.5.0+77 21 kB/s | 14 MB 10:56 2023-09-22T12:56:57.262 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:56:57.262 INFO:teuthology.orchestra.run.smithi191.stdout:Total 45 kB/s | 37 MB 13:44 2023-09-22T12:56:57.357 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-09-22T12:56:57.491 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T12:56:57.510 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T12:56:57.511 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T12:56:57.588 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-09-22T12:56:57.850 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T12:56:57.854 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T12:56:57.888 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-09-22T12:56:58.130 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-09-22T12:56:58.417 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T12:56:58.549 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T12:56:58.719 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T12:56:58.857 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T12:56:59.158 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-09-22T12:56:59.425 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-09-22T12:56:59.454 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T12:56:59.624 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T12:56:59.625 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-09-22T12:56:59.809 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-09-22T12:56:59.960 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T12:57:00.249 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T12:57:00.255 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T12:57:00.255 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 1/13 2023-09-22T12:57:00.255 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-09-22T12:57:00.255 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-09-22T12:57:00.255 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-09-22T12:57:00.255 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-09-22T12:57:00.255 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-22T12:57:00.255 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-22T12:57:00.256 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-09-22T12:57:00.256 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-22T12:57:00.256 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-09-22T12:57:00.256 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-09-22T12:57:00.256 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-09-22T12:57:00.391 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T12:57:00.924 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-09-22T12:57:00.924 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:57:00.924 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T12:57:00.924 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:57:00.924 INFO:teuthology.orchestra.run.smithi170.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-22T12:57:00.925 INFO:teuthology.orchestra.run.smithi170.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-22T12:57:00.925 INFO:teuthology.orchestra.run.smithi170.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-22T12:57:00.925 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-22T12:57:00.925 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-22T12:57:00.925 INFO:teuthology.orchestra.run.smithi170.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-22T12:57:00.925 INFO:teuthology.orchestra.run.smithi170.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-22T12:57:00.925 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-22T12:57:00.925 INFO:teuthology.orchestra.run.smithi170.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-22T12:57:00.925 INFO:teuthology.orchestra.run.smithi170.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-22T12:57:00.926 INFO:teuthology.orchestra.run.smithi170.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-22T12:57:00.926 INFO:teuthology.orchestra.run.smithi170.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-22T12:57:00.926 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:57:00.926 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T12:57:01.042 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install ceph-mgr-cephadm 2023-09-22T12:57:01.092 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T12:57:01.461 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T12:57:01.707 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T12:57:01.896 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:16:13 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:57:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T12:57:03.474 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T12:57:03.475 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:57:03.475 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repository Size 2023-09-22T12:57:03.475 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:57:03.476 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T12:57:03.476 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr-cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 116 k 2023-09-22T12:57:03.476 INFO:teuthology.orchestra.run.smithi170.stdout:Installing dependencies: 2023-09-22T12:57:03.476 INFO:teuthology.orchestra.run.smithi170.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-09-22T12:57:03.476 INFO:teuthology.orchestra.run.smithi170.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-09-22T12:57:03.476 INFO:teuthology.orchestra.run.smithi170.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-09-22T12:57:03.476 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-09-22T12:57:03.476 INFO:teuthology.orchestra.run.smithi170.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-09-22T12:57:03.476 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:57:03.476 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T12:57:03.476 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T12:57:03.477 INFO:teuthology.orchestra.run.smithi170.stdout:Install 6 Packages 2023-09-22T12:57:03.477 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T12:57:03.477 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 5.8 M 2023-09-22T12:57:03.477 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 24 M 2023-09-22T12:57:03.477 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T12:57:04.783 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-09-22T12:57:04.973 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T12:57:05.293 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T12:57:05.293 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 1/8 2023-09-22T12:57:05.293 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-09-22T12:57:05.293 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-09-22T12:57:05.294 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-09-22T12:57:05.294 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T12:57:05.294 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-09-22T12:57:05.294 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 7/8 2023-09-22T12:57:05.459 INFO:teuthology.orchestra.run.smithi170.stdout:(1/6): ceph-mgr-cephadm-16.2.14-5.g31c126b7.el8 263 kB/s | 116 kB 00:00 2023-09-22T12:57:05.551 INFO:teuthology.orchestra.run.smithi170.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 232 kB/s | 21 kB 00:00 2023-09-22T12:57:05.684 INFO:teuthology.orchestra.run.smithi170.stdout:(3/6): python3-execnet-1.7.1-1.el8.noarch.rpm 2.6 MB/s | 355 kB 00:00 2023-09-22T12:57:05.751 INFO:teuthology.orchestra.run.smithi170.stdout:(4/6): python3-remoto-1.2.1-1.el8.noarch.rpm 733 kB/s | 49 kB 00:00 2023-09-22T12:57:05.949 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 8/8 2023-09-22T12:57:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:57:05.950 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T12:57:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:57:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: libgfortran-8.5.0-20.el8.x86_64 2023-09-22T12:57:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: libquadmath-8.5.0-20.el8.x86_64 2023-09-22T12:57:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-0.3.15-4.el8.x86_64 2023-09-22T12:57:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-09-22T12:57:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-22T12:57:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-22T12:57:05.950 INFO:teuthology.orchestra.run.smithi191.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-09-22T12:57:05.951 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:57:05.951 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T12:57:06.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-rook 2023-09-22T12:57:06.934 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:16:18 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:57:08.504 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T12:57:08.506 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:57:08.506 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-09-22T12:57:08.506 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:57:08.507 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T12:57:08.507 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-rook noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 27 k 2023-09-22T12:57:08.507 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-09-22T12:57:08.507 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-22T12:57:08.507 INFO:teuthology.orchestra.run.smithi191.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-22T12:57:08.507 INFO:teuthology.orchestra.run.smithi191.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-22T12:57:08.507 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-09-22T12:57:08.508 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-09-22T12:57:08.508 INFO:teuthology.orchestra.run.smithi191.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-22T12:57:08.508 INFO:teuthology.orchestra.run.smithi191.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-09-22T12:57:08.508 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-09-22T12:57:08.508 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-09-22T12:57:08.508 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-09-22T12:57:08.508 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-22T12:57:08.509 INFO:teuthology.orchestra.run.smithi191.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-22T12:57:08.509 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:57:08.509 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T12:57:08.509 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:57:08.509 INFO:teuthology.orchestra.run.smithi191.stdout:Install 13 Packages 2023-09-22T12:57:08.509 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:57:08.510 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 1.9 M 2023-09-22T12:57:08.510 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 24 M 2023-09-22T12:57:08.510 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T12:57:11.385 INFO:teuthology.orchestra.run.smithi191.stdout:(1/13): ceph-mgr-rook-16.2.14-5.g31c126b7.el8.n 61 kB/s | 27 kB 00:00 2023-09-22T12:57:12.768 INFO:teuthology.orchestra.run.smithi191.stdout:(2/13): python3-jsonpointer-1.10-11.el8.noarch. 11 kB/s | 20 kB 00:01 2023-09-22T12:57:14.977 INFO:teuthology.orchestra.run.smithi191.stdout:(3/13): python3-pyasn1-modules-0.3.7-6.el8.noar 50 kB/s | 110 kB 00:02 2023-09-22T12:57:15.335 INFO:teuthology.orchestra.run.smithi191.stdout:(4/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 432 kB/s | 155 kB 00:00 2023-09-22T12:57:15.419 INFO:teuthology.orchestra.run.smithi191.stdout:(5/13): python3-requests-oauthlib-1.0.0-1.el8.n 511 kB/s | 43 kB 00:00 2023-09-22T12:57:15.603 INFO:teuthology.orchestra.run.smithi191.stdout:(6/13): python3-cachetools-3.1.1-4.el8.noarch.r 180 kB/s | 33 kB 00:00 2023-09-22T12:57:15.703 INFO:teuthology.orchestra.run.smithi191.stdout:(7/13): python3-certifi-2018.10.15-7.el8.noarch 163 kB/s | 16 kB 00:00 2023-09-22T12:57:15.770 INFO:teuthology.orchestra.run.smithi191.stdout:(8/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 29 kB/s | 126 kB 00:04 2023-09-22T12:57:15.854 INFO:teuthology.orchestra.run.smithi191.stdout:(9/13): python3-google-auth-1.1.1-10.el8.noarch 539 kB/s | 81 kB 00:00 2023-09-22T12:57:15.971 INFO:teuthology.orchestra.run.smithi191.stdout:(10/13): python3-rsa-4.9-2.el8.noarch.rpm 549 kB/s | 64 kB 00:00 2023-09-22T12:57:16.097 INFO:teuthology.orchestra.run.smithi191.stdout:(11/13): python3-websocket-client-0.56.0-5.el8. 489 kB/s | 61 kB 00:00 2023-09-22T12:57:16.155 INFO:teuthology.orchestra.run.smithi191.stdout:(12/13): python3-jsonpatch-1.21-2.el8.noarch.rp 5.1 kB/s | 27 kB 00:05 2023-09-22T12:57:16.239 INFO:teuthology.orchestra.run.smithi191.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 2.5 MB/s | 1.2 MB 00:00 2023-09-22T12:57:16.240 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:57:16.240 INFO:teuthology.orchestra.run.smithi191.stdout:Total 253 kB/s | 1.9 MB 00:07 2023-09-22T12:57:16.299 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T12:57:16.315 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T12:57:16.315 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T12:57:16.404 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T12:57:16.406 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T12:57:16.885 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T12:57:17.132 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-22T12:57:17.329 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-09-22T12:57:17.564 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-09-22T12:57:17.769 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-09-22T12:57:17.976 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-09-22T12:57:18.207 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-09-22T12:57:18.498 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-09-22T12:57:18.741 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-09-22T12:57:19.781 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-09-22T12:57:20.127 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-09-22T12:57:20.311 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-09-22T12:57:20.494 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-09-22T12:57:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T12:57:20.951 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T12:57:20.951 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 1/13 2023-09-22T12:57:20.951 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-09-22T12:57:20.952 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-09-22T12:57:20.952 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-09-22T12:57:20.952 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-09-22T12:57:20.952 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-22T12:57:20.952 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-22T12:57:20.952 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-09-22T12:57:20.952 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-22T12:57:20.952 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-09-22T12:57:20.952 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-09-22T12:57:20.952 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-09-22T12:57:21.633 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-09-22T12:57:21.633 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:57:21.634 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T12:57:21.634 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:57:21.634 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-22T12:57:21.634 INFO:teuthology.orchestra.run.smithi191.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-22T12:57:21.634 INFO:teuthology.orchestra.run.smithi191.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-22T12:57:21.634 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-22T12:57:21.634 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-22T12:57:21.634 INFO:teuthology.orchestra.run.smithi191.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-22T12:57:21.634 INFO:teuthology.orchestra.run.smithi191.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-22T12:57:21.634 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-22T12:57:21.634 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-22T12:57:21.635 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-22T12:57:21.635 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-22T12:57:21.635 INFO:teuthology.orchestra.run.smithi191.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-22T12:57:21.635 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:57:21.635 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T12:57:21.760 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-cephadm 2023-09-22T12:57:22.617 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:16:34 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T12:57:24.183 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T12:57:24.184 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:57:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-09-22T12:57:24.185 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:57:24.185 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T12:57:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 116 k 2023-09-22T12:57:24.185 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-09-22T12:57:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-09-22T12:57:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-09-22T12:57:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-09-22T12:57:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-09-22T12:57:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-09-22T12:57:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:57:24.186 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T12:57:24.186 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T12:57:24.187 INFO:teuthology.orchestra.run.smithi191.stdout:Install 6 Packages 2023-09-22T12:57:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T12:57:24.187 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 5.8 M 2023-09-22T12:57:24.188 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 24 M 2023-09-22T12:57:24.188 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T12:57:24.626 INFO:teuthology.orchestra.run.smithi191.stdout:(1/6): ceph-mgr-cephadm-16.2.14-5.g31c126b7.el8 614 kB/s | 116 kB 00:00 2023-09-22T12:57:24.877 INFO:teuthology.orchestra.run.smithi191.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 85 kB/s | 21 kB 00:00 2023-09-22T12:57:25.285 INFO:teuthology.orchestra.run.smithi191.stdout:(3/6): python3-execnet-1.7.1-1.el8.noarch.rpm 870 kB/s | 355 kB 00:00 2023-09-22T12:57:25.386 INFO:teuthology.orchestra.run.smithi191.stdout:(4/6): python3-remoto-1.2.1-1.el8.noarch.rpm 487 kB/s | 49 kB 00:00 2023-09-22T12:57:28.328 INFO:teuthology.orchestra.run.smithi170.stdout:(5/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 23 kB/s | 538 kB 00:23 2023-09-22T12:57:47.689 INFO:teuthology.orchestra.run.smithi191.stdout:(5/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 23 kB/s | 538 kB 00:23 2023-09-22T13:00:18.947 INFO:teuthology.orchestra.run.smithi170.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 25 kB/s | 4.8 MB 03:13 2023-09-22T13:00:18.948 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:00:18.949 INFO:teuthology.orchestra.run.smithi170.stdout:Total 31 kB/s | 5.8 MB 03:15 2023-09-22T13:00:19.001 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T13:00:19.011 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T13:00:19.011 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T13:00:19.118 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T13:00:19.121 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T13:00:19.451 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T13:00:19.693 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 1/6 2023-09-22T13:00:19.954 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 2/6 2023-09-22T13:00:20.896 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 3/6 2023-09-22T13:00:21.240 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 4/6 2023-09-22T13:00:21.518 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-09-22T13:00:21.676 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:00:21.953 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:00:21.953 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/6 2023-09-22T13:00:21.954 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/6 2023-09-22T13:00:21.954 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/6 2023-09-22T13:00:21.954 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 4/6 2023-09-22T13:00:21.954 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 5/6 2023-09-22T13:00:22.574 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 6/6 2023-09-22T13:00:22.574 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:22.575 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T13:00:22.575 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:00:22.575 INFO:teuthology.orchestra.run.smithi170.stdout: python3-apipkg-1.5-6.el8.noarch 2023-09-22T13:00:22.575 INFO:teuthology.orchestra.run.smithi170.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-22T13:00:22.575 INFO:teuthology.orchestra.run.smithi170.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-09-22T13:00:22.575 INFO:teuthology.orchestra.run.smithi170.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-22T13:00:22.575 INFO:teuthology.orchestra.run.smithi170.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-09-22T13:00:22.575 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:22.576 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:22.674 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install ceph-fuse 2023-09-22T13:00:23.532 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:19:35 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:25.120 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:25.120 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:25.121 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repository Size 2023-09-22T13:00:25.121 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:25.121 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T13:00:25.121 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 910 k 2023-09-22T13:00:25.121 INFO:teuthology.orchestra.run.smithi170.stdout:Installing dependencies: 2023-09-22T13:00:25.121 INFO:teuthology.orchestra.run.smithi170.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-09-22T13:00:25.121 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:25.122 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T13:00:25.122 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:25.122 INFO:teuthology.orchestra.run.smithi170.stdout:Install 2 Packages 2023-09-22T13:00:25.122 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:25.123 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 993 k 2023-09-22T13:00:25.123 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 2.8 M 2023-09-22T13:00:25.123 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T13:00:25.464 INFO:teuthology.orchestra.run.smithi170.stdout:(1/2): fuse-2.9.7-17.el8.x86_64.rpm 278 kB/s | 83 kB 00:00 2023-09-22T13:00:25.697 INFO:teuthology.orchestra.run.smithi170.stdout:(2/2): ceph-fuse-16.2.14-5.g31c126b7.el8.x86_64 1.7 MB/s | 910 kB 00:00 2023-09-22T13:00:25.698 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:00:25.698 INFO:teuthology.orchestra.run.smithi170.stdout:Total 1.7 MB/s | 993 kB 00:00 2023-09-22T13:00:25.709 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T13:00:25.729 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T13:00:25.729 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T13:00:25.819 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T13:00:25.822 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T13:00:25.990 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T13:00:26.251 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-09-22T13:00:26.434 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:00:26.823 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:00:26.823 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/2 2023-09-22T13:00:27.465 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-09-22T13:00:27.465 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:27.465 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T13:00:27.466 INFO:teuthology.orchestra.run.smithi170.stdout: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-09-22T13:00:27.466 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:27.466 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:27.675 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install librados-devel 2023-09-22T13:00:28.530 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:19:40 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:30.106 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:30.106 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:30.106 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repo Size 2023-09-22T13:00:30.106 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:30.106 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T13:00:30.106 INFO:teuthology.orchestra.run.smithi170.stdout: librados-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 131 k 2023-09-22T13:00:30.107 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:30.107 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T13:00:30.107 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:30.107 INFO:teuthology.orchestra.run.smithi170.stdout:Install 1 Package 2023-09-22T13:00:30.107 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:30.107 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 131 k 2023-09-22T13:00:30.108 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 432 k 2023-09-22T13:00:30.108 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T13:00:30.309 INFO:teuthology.orchestra.run.smithi170.stdout:librados-devel-16.2.14-5.g31c126b7.el8.x86_64.r 651 kB/s | 131 kB 00:00 2023-09-22T13:00:30.309 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:00:30.310 INFO:teuthology.orchestra.run.smithi170.stdout:Total 646 kB/s | 131 kB 00:00 2023-09-22T13:00:30.310 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T13:00:30.315 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T13:00:30.315 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T13:00:30.365 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T13:00:30.368 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T13:00:30.479 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T13:00:30.683 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:00:30.976 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:00:31.590 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:00:31.590 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:31.590 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T13:00:31.591 INFO:teuthology.orchestra.run.smithi170.stdout: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:00:31.591 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:31.591 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:31.783 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install libcephfs2 2023-09-22T13:00:32.641 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:19:44 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:34.036 INFO:teuthology.orchestra.run.smithi170.stdout:Package libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:00:34.219 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:34.220 INFO:teuthology.orchestra.run.smithi170.stdout:Nothing to do. 2023-09-22T13:00:34.220 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:34.317 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install libcephfs-devel 2023-09-22T13:00:35.182 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:19:47 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:36.775 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:36.775 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:36.775 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repo Size 2023-09-22T13:00:36.775 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:36.775 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T13:00:36.775 INFO:teuthology.orchestra.run.smithi170.stdout: libcephfs-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 25 k 2023-09-22T13:00:36.776 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:36.776 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T13:00:36.776 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:36.776 INFO:teuthology.orchestra.run.smithi170.stdout:Install 1 Package 2023-09-22T13:00:36.776 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:36.776 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 25 k 2023-09-22T13:00:36.776 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 107 k 2023-09-22T13:00:36.777 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T13:00:36.929 INFO:teuthology.orchestra.run.smithi170.stdout:libcephfs-devel-16.2.14-5.g31c126b7.el8.x86_64. 167 kB/s | 25 kB 00:00 2023-09-22T13:00:36.929 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:00:36.930 INFO:teuthology.orchestra.run.smithi170.stdout:Total 166 kB/s | 25 kB 00:00 2023-09-22T13:00:36.930 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T13:00:36.933 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T13:00:36.933 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T13:00:36.936 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T13:00:36.939 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T13:00:36.986 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T13:00:37.181 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:00:37.442 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:00:38.001 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:00:38.001 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:38.001 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T13:00:38.002 INFO:teuthology.orchestra.run.smithi170.stdout: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:00:38.002 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:38.002 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:38.114 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install librados2 2023-09-22T13:00:38.969 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:19:50 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:40.378 INFO:teuthology.orchestra.run.smithi170.stdout:Package librados2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:00:40.557 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:40.558 INFO:teuthology.orchestra.run.smithi170.stdout:Nothing to do. 2023-09-22T13:00:40.558 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:40.654 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install librbd1 2023-09-22T13:00:41.513 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:19:53 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:42.906 INFO:teuthology.orchestra.run.smithi170.stdout:Package librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:00:43.088 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:43.089 INFO:teuthology.orchestra.run.smithi170.stdout:Nothing to do. 2023-09-22T13:00:43.089 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:43.185 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install python3-rados 2023-09-22T13:00:44.038 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:19:56 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:45.432 INFO:teuthology.orchestra.run.smithi170.stdout:Package python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:00:45.609 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:45.611 INFO:teuthology.orchestra.run.smithi170.stdout:Nothing to do. 2023-09-22T13:00:45.611 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:45.704 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install python3-rgw 2023-09-22T13:00:46.560 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:19:58 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:47.968 INFO:teuthology.orchestra.run.smithi170.stdout:Package python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:00:48.148 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:48.149 INFO:teuthology.orchestra.run.smithi170.stdout:Nothing to do. 2023-09-22T13:00:48.149 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:48.243 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install python3-cephfs 2023-09-22T13:00:49.103 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:20:01 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:50.497 INFO:teuthology.orchestra.run.smithi170.stdout:Package python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:00:50.682 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:50.684 INFO:teuthology.orchestra.run.smithi170.stdout:Nothing to do. 2023-09-22T13:00:50.684 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:50.780 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install python3-rbd 2023-09-22T13:00:51.631 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:20:03 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:53.009 INFO:teuthology.orchestra.run.smithi170.stdout:Package python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:00:53.190 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:53.191 INFO:teuthology.orchestra.run.smithi170.stdout:Nothing to do. 2023-09-22T13:00:53.191 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:53.286 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install rbd-fuse 2023-09-22T13:00:54.140 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:20:06 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:55.717 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:55.718 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:55.718 INFO:teuthology.orchestra.run.smithi170.stdout: Package Architecture Version Repository Size 2023-09-22T13:00:55.718 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:55.718 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T13:00:55.719 INFO:teuthology.orchestra.run.smithi170.stdout: rbd-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 88 k 2023-09-22T13:00:55.719 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:55.719 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T13:00:55.719 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:55.719 INFO:teuthology.orchestra.run.smithi170.stdout:Install 1 Package 2023-09-22T13:00:55.719 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:55.720 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 88 k 2023-09-22T13:00:55.720 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 240 k 2023-09-22T13:00:55.720 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T13:00:55.882 INFO:teuthology.orchestra.run.smithi170.stdout:rbd-fuse-16.2.14-5.g31c126b7.el8.x86_64.rpm 548 kB/s | 88 kB 00:00 2023-09-22T13:00:55.882 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:00:55.883 INFO:teuthology.orchestra.run.smithi170.stdout:Total 544 kB/s | 88 kB 00:00 2023-09-22T13:00:55.883 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T13:00:55.893 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T13:00:55.893 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T13:00:55.941 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T13:00:55.945 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T13:00:56.045 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T13:00:56.245 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:00:56.540 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:00:57.184 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:00:57.184 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:57.184 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T13:00:57.184 INFO:teuthology.orchestra.run.smithi170.stdout: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:00:57.184 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:57.185 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:00:57.365 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install rbd-mirror 2023-09-22T13:00:58.218 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:20:10 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:00:59.791 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:00:59.792 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:59.792 INFO:teuthology.orchestra.run.smithi170.stdout: Package Architecture Version Repository Size 2023-09-22T13:00:59.792 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:59.792 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T13:00:59.792 INFO:teuthology.orchestra.run.smithi170.stdout: rbd-mirror x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T13:00:59.792 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:59.792 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T13:00:59.792 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:00:59.792 INFO:teuthology.orchestra.run.smithi170.stdout:Install 1 Package 2023-09-22T13:00:59.792 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:00:59.793 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 3.7 M 2023-09-22T13:00:59.793 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 15 M 2023-09-22T13:00:59.793 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T13:01:00.402 INFO:teuthology.orchestra.run.smithi170.stdout:rbd-mirror-16.2.14-5.g31c126b7.el8.x86_64.rpm 6.0 MB/s | 3.7 MB 00:00 2023-09-22T13:01:00.403 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:00.403 INFO:teuthology.orchestra.run.smithi170.stdout:Total 6.0 MB/s | 3.7 MB 00:00 2023-09-22T13:01:00.403 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T13:01:00.415 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T13:01:00.415 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T13:01:00.478 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T13:01:00.481 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T13:01:00.738 INFO:teuthology.orchestra.run.smithi191.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 23 kB/s | 4.8 MB 03:36 2023-09-22T13:01:00.740 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:00.740 INFO:teuthology.orchestra.run.smithi191.stdout:Total 28 kB/s | 5.8 MB 03:36 2023-09-22T13:01:00.791 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T13:01:00.802 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T13:01:00.802 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T13:01:00.916 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T13:01:00.919 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T13:01:00.927 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T13:01:01.129 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:01.236 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T13:01:01.512 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 1/6 2023-09-22T13:01:01.561 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:01.791 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 2/6 2023-09-22T13:01:02.176 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:02.176 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:02.177 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T13:01:02.177 INFO:teuthology.orchestra.run.smithi170.stdout: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:01:02.177 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:02.177 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:01:02.384 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum -y install rbd-nbd 2023-09-22T13:01:02.716 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 3/6 2023-09-22T13:01:03.059 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 4/6 2023-09-22T13:01:03.242 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:20:15 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:03.304 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-09-22T13:01:03.546 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:01:03.830 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:01:03.830 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/6 2023-09-22T13:01:03.830 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/6 2023-09-22T13:01:03.830 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/6 2023-09-22T13:01:03.830 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 4/6 2023-09-22T13:01:03.830 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 5/6 2023-09-22T13:01:04.460 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 6/6 2023-09-22T13:01:04.460 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:04.460 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T13:01:04.460 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:01:04.460 INFO:teuthology.orchestra.run.smithi191.stdout: python3-apipkg-1.5-6.el8.noarch 2023-09-22T13:01:04.460 INFO:teuthology.orchestra.run.smithi191.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-22T13:01:04.461 INFO:teuthology.orchestra.run.smithi191.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-09-22T13:01:04.461 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-22T13:01:04.461 INFO:teuthology.orchestra.run.smithi191.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-09-22T13:01:04.461 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:04.461 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:04.569 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-fuse 2023-09-22T13:01:04.819 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:01:04.820 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:01:04.820 INFO:teuthology.orchestra.run.smithi170.stdout: Package Architecture Version Repository Size 2023-09-22T13:01:04.820 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:01:04.821 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T13:01:04.821 INFO:teuthology.orchestra.run.smithi170.stdout: rbd-nbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 173 k 2023-09-22T13:01:04.821 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:04.821 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T13:01:04.821 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:01:04.821 INFO:teuthology.orchestra.run.smithi170.stdout:Install 1 Package 2023-09-22T13:01:04.821 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:04.821 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 173 k 2023-09-22T13:01:04.822 INFO:teuthology.orchestra.run.smithi170.stdout:Installed size: 515 k 2023-09-22T13:01:04.822 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T13:01:05.155 INFO:teuthology.orchestra.run.smithi170.stdout:rbd-nbd-16.2.14-5.g31c126b7.el8.x86_64.rpm 518 kB/s | 173 kB 00:00 2023-09-22T13:01:05.156 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:05.156 INFO:teuthology.orchestra.run.smithi170.stdout:Total 516 kB/s | 173 kB 00:00 2023-09-22T13:01:05.156 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T13:01:05.167 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T13:01:05.167 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T13:01:05.216 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T13:01:05.219 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T13:01:05.327 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T13:01:05.427 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:17 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:05.493 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:05.775 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:06.387 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:06.387 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:06.387 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T13:01:06.388 INFO:teuthology.orchestra.run.smithi170.stdout: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:01:06.388 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:06.388 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:01:06.585 DEBUG:teuthology.parallel:result is None 2023-09-22T13:01:07.003 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:07.004 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:07.004 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2023-09-22T13:01:07.004 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:07.004 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T13:01:07.005 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 910 k 2023-09-22T13:01:07.005 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2023-09-22T13:01:07.005 INFO:teuthology.orchestra.run.smithi191.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-09-22T13:01:07.005 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:07.005 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T13:01:07.005 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:07.005 INFO:teuthology.orchestra.run.smithi191.stdout:Install 2 Packages 2023-09-22T13:01:07.005 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:07.006 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 993 k 2023-09-22T13:01:07.006 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 2.8 M 2023-09-22T13:01:07.006 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T13:01:07.559 INFO:teuthology.orchestra.run.smithi191.stdout:(1/2): ceph-fuse-16.2.14-5.g31c126b7.el8.x86_64 2.7 MB/s | 910 kB 00:00 2023-09-22T13:01:07.934 INFO:teuthology.orchestra.run.smithi191.stdout:(2/2): fuse-2.9.7-17.el8.x86_64.rpm 118 kB/s | 83 kB 00:00 2023-09-22T13:01:07.935 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:07.935 INFO:teuthology.orchestra.run.smithi191.stdout:Total 1.0 MB/s | 993 kB 00:00 2023-09-22T13:01:07.947 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T13:01:07.968 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T13:01:07.968 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T13:01:08.058 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T13:01:08.061 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T13:01:08.231 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T13:01:08.495 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-09-22T13:01:08.712 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:01:09.158 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:01:09.158 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/2 2023-09-22T13:01:09.811 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-09-22T13:01:09.811 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:09.811 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T13:01:09.811 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-09-22T13:01:09.811 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:09.812 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:10.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install librados-devel 2023-09-22T13:01:10.888 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:22 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:12.462 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:12.463 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:12.463 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repo Size 2023-09-22T13:01:12.463 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:12.464 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T13:01:12.464 INFO:teuthology.orchestra.run.smithi191.stdout: librados-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 131 k 2023-09-22T13:01:12.464 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:12.464 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T13:01:12.464 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:12.464 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-09-22T13:01:12.464 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:12.464 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 131 k 2023-09-22T13:01:12.465 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 432 k 2023-09-22T13:01:12.465 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T13:01:12.709 INFO:teuthology.orchestra.run.smithi191.stdout:librados-devel-16.2.14-5.g31c126b7.el8.x86_64.r 534 kB/s | 131 kB 00:00 2023-09-22T13:01:12.710 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:12.710 INFO:teuthology.orchestra.run.smithi191.stdout:Total 532 kB/s | 131 kB 00:00 2023-09-22T13:01:12.710 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T13:01:12.715 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T13:01:12.715 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T13:01:12.765 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T13:01:12.768 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T13:01:12.910 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T13:01:13.127 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:13.424 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:14.035 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:14.036 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:14.036 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T13:01:14.036 INFO:teuthology.orchestra.run.smithi191.stdout: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:01:14.036 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:14.036 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:14.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install libcephfs2 2023-09-22T13:01:15.126 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:27 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:16.522 INFO:teuthology.orchestra.run.smithi191.stdout:Package libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:01:16.702 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:16.703 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-09-22T13:01:16.703 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:16.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install libcephfs-devel 2023-09-22T13:01:17.677 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:29 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:19.250 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:19.251 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:19.252 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repo Size 2023-09-22T13:01:19.252 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:19.252 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T13:01:19.252 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 25 k 2023-09-22T13:01:19.252 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:19.252 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T13:01:19.252 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:19.252 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-09-22T13:01:19.253 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:19.253 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 25 k 2023-09-22T13:01:19.253 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 107 k 2023-09-22T13:01:19.253 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T13:01:19.407 INFO:teuthology.orchestra.run.smithi191.stdout:libcephfs-devel-16.2.14-5.g31c126b7.el8.x86_64. 164 kB/s | 25 kB 00:00 2023-09-22T13:01:19.408 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:19.408 INFO:teuthology.orchestra.run.smithi191.stdout:Total 163 kB/s | 25 kB 00:00 2023-09-22T13:01:19.408 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T13:01:19.412 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T13:01:19.412 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T13:01:19.414 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T13:01:19.417 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T13:01:19.469 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T13:01:19.717 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:19.974 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:20.570 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:20.571 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:20.571 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T13:01:20.571 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:01:20.571 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:20.571 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:20.686 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install librados2 2023-09-22T13:01:21.544 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:33 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:22.931 INFO:teuthology.orchestra.run.smithi191.stdout:Package librados2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:01:23.113 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:23.114 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-09-22T13:01:23.115 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:23.221 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install librbd1 2023-09-22T13:01:24.082 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:36 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:25.473 INFO:teuthology.orchestra.run.smithi191.stdout:Package librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:01:25.653 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:25.654 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-09-22T13:01:25.655 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:25.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-rados 2023-09-22T13:01:26.615 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:38 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:28.010 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:01:28.190 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:28.192 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-09-22T13:01:28.192 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:28.302 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-rgw 2023-09-22T13:01:29.160 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:41 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:30.570 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:01:30.753 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:30.754 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-09-22T13:01:30.754 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:30.886 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-cephfs 2023-09-22T13:01:31.742 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:43 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:33.136 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:01:33.317 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:33.318 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-09-22T13:01:33.319 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:33.431 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-rbd 2023-09-22T13:01:34.283 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:46 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:35.678 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:01:35.862 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:35.863 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2023-09-22T13:01:35.863 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:35.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install rbd-fuse 2023-09-22T13:01:36.832 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:48 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:38.392 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:38.393 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:38.393 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2023-09-22T13:01:38.393 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:38.393 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T13:01:38.393 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 88 k 2023-09-22T13:01:38.393 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:38.393 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T13:01:38.393 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:38.393 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-09-22T13:01:38.393 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:38.394 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 88 k 2023-09-22T13:01:38.394 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 240 k 2023-09-22T13:01:38.394 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T13:01:38.687 INFO:teuthology.orchestra.run.smithi191.stdout:rbd-fuse-16.2.14-5.g31c126b7.el8.x86_64.rpm 301 kB/s | 88 kB 00:00 2023-09-22T13:01:38.688 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:38.688 INFO:teuthology.orchestra.run.smithi191.stdout:Total 300 kB/s | 88 kB 00:00 2023-09-22T13:01:38.689 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T13:01:38.698 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T13:01:38.698 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T13:01:38.749 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T13:01:38.752 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T13:01:38.851 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T13:01:39.039 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:39.331 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:39.932 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T13:01:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:01:39.932 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:39.932 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:40.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install rbd-mirror 2023-09-22T13:01:41.088 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:53 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:42.636 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:42.637 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:42.637 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2023-09-22T13:01:42.637 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:42.637 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T13:01:42.637 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-mirror x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T13:01:42.637 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:42.637 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T13:01:42.638 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:42.638 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-09-22T13:01:42.638 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:42.638 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 3.7 M 2023-09-22T13:01:42.638 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 15 M 2023-09-22T13:01:42.638 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T13:01:44.128 INFO:teuthology.orchestra.run.smithi191.stdout:rbd-mirror-16.2.14-5.g31c126b7.el8.x86_64.rpm 2.5 MB/s | 3.7 MB 00:01 2023-09-22T13:01:44.129 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:44.130 INFO:teuthology.orchestra.run.smithi191.stdout:Total 2.5 MB/s | 3.7 MB 00:01 2023-09-22T13:01:44.130 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T13:01:44.142 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T13:01:44.142 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T13:01:44.209 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T13:01:44.212 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T13:01:44.670 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T13:01:44.873 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:45.300 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:45.906 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:45.907 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:45.907 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T13:01:45.907 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:01:45.907 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:45.907 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:46.120 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install rbd-nbd 2023-09-22T13:01:46.983 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:20:58 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:01:48.541 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2023-09-22T13:01:48.543 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:48.543 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2023-09-22T13:01:48.543 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:48.543 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2023-09-22T13:01:48.543 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-nbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 173 k 2023-09-22T13:01:48.543 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:48.543 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2023-09-22T13:01:48.544 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2023-09-22T13:01:48.544 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2023-09-22T13:01:48.544 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:48.544 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 173 k 2023-09-22T13:01:48.544 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 515 k 2023-09-22T13:01:48.544 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2023-09-22T13:01:48.792 INFO:teuthology.orchestra.run.smithi191.stdout:rbd-nbd-16.2.14-5.g31c126b7.el8.x86_64.rpm 697 kB/s | 173 kB 00:00 2023-09-22T13:01:48.793 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:48.793 INFO:teuthology.orchestra.run.smithi191.stdout:Total 692 kB/s | 173 kB 00:00 2023-09-22T13:01:48.793 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2023-09-22T13:01:48.804 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2023-09-22T13:01:48.804 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2023-09-22T13:01:48.854 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2023-09-22T13:01:48.858 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2023-09-22T13:01:48.967 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2023-09-22T13:01:49.170 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:49.465 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:50.078 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:01:50.078 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:50.078 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2023-09-22T13:01:50.078 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:01:50.078 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:50.079 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2023-09-22T13:01:50.258 DEBUG:teuthology.parallel:result is None 2023-09-22T13:01:50.259 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:01:50.528 DEBUG:teuthology.orchestra.run.smithi170:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-22T13:01:50.557 INFO:teuthology.orchestra.run.smithi170.stdout:16.2.14-5.g31c126b7.el8 2023-09-22T13:01:50.557 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-5.g31c126b7.el8 2023-09-22T13:01:50.557 INFO:teuthology.task.install:The correct ceph version 16.2.14-5.g31c126b7 is installed. 2023-09-22T13:01:50.558 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:01:50.723 DEBUG:teuthology.orchestra.run.smithi191:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-22T13:01:50.747 INFO:teuthology.orchestra.run.smithi191.stdout:16.2.14-5.g31c126b7.el8 2023-09-22T13:01:50.747 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-5.g31c126b7.el8 2023-09-22T13:01:50.747 INFO:teuthology.task.install:The correct ceph version 16.2.14-5.g31c126b7 is installed. 2023-09-22T13:01:50.747 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-22T13:01:50.748 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:01:50.748 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-22T13:01:50.783 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:01:50.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-22T13:01:50.816 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-22T13:01:50.816 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:01:50.816 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd of=/usr/bin/daemon-helper 2023-09-22T13:01:50.855 DEBUG:teuthology.orchestra.run.smithi170:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-22T13:01:50.925 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:01:50.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/daemon-helper 2023-09-22T13:01:50.954 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-22T13:01:51.021 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-22T13:01:51.021 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:01:51.021 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-22T13:01:51.051 DEBUG:teuthology.orchestra.run.smithi170:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-22T13:01:51.119 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:01:51.120 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-22T13:01:51.147 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-22T13:01:51.214 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-22T13:01:51.215 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:01:51.215 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd of=/usr/bin/stdin-killer 2023-09-22T13:01:51.243 DEBUG:teuthology.orchestra.run.smithi170:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-22T13:01:51.312 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:01:51.312 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/stdin-killer 2023-09-22T13:01:51.339 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-22T13:01:51.407 INFO:teuthology.run_tasks:Running task ceph... 2023-09-22T13:01:51.469 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2023-09-22T13:01:51.469 DEBUG:teuthology.orchestra.run.smithi170:> sudo chmod 777 /var/log/ceph 2023-09-22T13:01:51.472 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod 777 /var/log/ceph 2023-09-22T13:01:51.499 INFO:tasks.ceph:Disabling ceph logrotate... 2023-09-22T13:01:51.499 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -f -- /etc/logrotate.d/ceph 2023-09-22T13:01:51.540 DEBUG:teuthology.orchestra.run.smithi191:> sudo rm -f -- /etc/logrotate.d/ceph 2023-09-22T13:01:51.567 INFO:tasks.ceph:Creating extra log directories... 2023-09-22T13:01:51.567 DEBUG:teuthology.orchestra.run.smithi170:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2023-09-22T13:01:51.608 DEBUG:teuthology.orchestra.run.smithi191:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2023-09-22T13:01:51.645 INFO:tasks.ceph:Creating ceph cluster ceph... 2023-09-22T13:01:51.645 INFO:tasks.ceph:config {'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=', 'rgw lc debug interval': 10, 'rgw max chunk size': 4194304, 'rgw obj stripe size': 6291456, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'global': {'mon client directed command retry': 5, 'ms inject socket failures': 5000, 'osd heartbeat grace': 40, 'osd_max_pg_log_entries': 10, 'osd_min_pg_log_entries': 10}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon osd crush smoke test': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd fast shutdown': False, 'osd objectstore': 'bluestore', 'osd shutdown pgref assert': True}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(POOL_APP_NOT_ENABLED\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'ceph', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2023-09-22T13:01:51.645 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400916', 'branch': 'wip-cfsnyder-bucket-check-pacific', 'description': 'rgw/verify/{centos_latest clusters/fixed-2 frontend/civetweb ignore-pg-availability msgr-failures/few objectstore/bluestore-bitmap overrides proto/https rgw_pool_type/ec sharding$/{default} striping$/{stripe-greater-than-chunk} tasks/{0-install bucket-check cls ragweed reshard s3tests-java s3tests versioning} validater/valgrind}', 'email': None, 'first_in_suite': False, 'job_id': '7400916', 'kernel': {'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'name': 'cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi', 'no_nested_subset': False, 'nuke-on-error': True, 'openstack': [{'volumes': {'count': 4, 'size': 10}}], 'os_type': 'centos', 'os_version': '8.stream', 'overrides': {'admin_socket': {'branch': 'wip-cfsnyder-bucket-check-pacific'}, 'ceph': {'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=', 'rgw lc debug interval': 10, 'rgw max chunk size': 4194304, 'rgw obj stripe size': 6291456, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'global': {'mon client directed command retry': 5, 'ms inject socket failures': 5000, 'osd heartbeat grace': 40, 'osd_max_pg_log_entries': 10, 'osd_min_pg_log_entries': 10}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon osd crush smoke test': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd fast shutdown': False, 'osd objectstore': 'bluestore', 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(POOL_APP_NOT_ENABLED\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {'osd default pool size': 2}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}}, 'openssl_keys': {'rgw.client.0': {'ca': 'root', 'client': 'client.0', 'embed-key': True}, 'rgw.client.1': {'ca': 'root', 'client': 'client.1', 'embed-key': True}, 'root': {'client': 'client.0', 'cn': 'teuthology', 'install': ['client.0', 'client.1'], 'key-type': 'rsa:4096'}}, 'rgw': {'client.0': {'ssl certificate': 'rgw.client.0'}, 'client.1': {'ssl certificate': 'rgw.client.1'}, 'compression type': 'random', 'ec-data-pool': True, 'frontend': 'civetweb', 'storage classes': 'LUKEWARM, FROZEN'}, 's3tests': {'slow_backend': True}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-cfsnyder-bucket-check-pacific', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}}, 'owner': 'scheduled_cfsnyder@teuthology', 'priority': 101, 'repo': 'https://git.ceph.com/ceph-ci.git', 'roles': [['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a'], ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'node-exporter.b']], 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'sleep_before_teardown': 0, 'suite': 'rgw', 'suite_branch': 'wip-cfsnyder-bucket-check-pacific', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph-ci.git', 'suite_sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'targets': {'smithi170.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGW1RWBfQqQYn2+WYnqrlGDOOPGyQm5QE4qsRK/5elTEgdVL6uLQVo5gdgU01NNAlnLIf5QJorVaKeIFPv6zhmg=', 'smithi191.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDh6tNYcj7aus9EYSbirgEOwv1/CZc8dW4p2jSWN9n/V1HkaO8NCSJSbt2T3vIm134ruxzApHaWCliSSjzp224s='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': None}, {'ceph': None}, {'openssl_keys': None}, {'rgw': {'client.0': None}}, {'workunit': {'clients': {'client.0': ['rgw/run-bucket-check.sh']}}}, {'workunit': {'clients': {'client.0': ['cls/test_cls_lock.sh', 'cls/test_cls_log.sh', 'cls/test_cls_refcount.sh', 'cls/test_cls_rgw.sh', 'cls/test_cls_cmpomap.sh', 'cls/test_cls_2pc_queue.sh', 'rgw/test_rgw_gc_log.sh', 'rgw/test_rgw_obj.sh', 'rgw/test_rgw_throttle.sh', 'rgw/test_librgw_file.sh']}}}, {'ragweed': {'client.0': {'default-branch': 'ceph-pacific', 'rgw_server': 'client.0', 'stages': 'prepare,check'}}}, {'workunit': {'clients': {'client.0': ['rgw/run-reshard.sh']}}}, {'s3tests-java': {'client.0': {'force-branch': 'ceph-pacific', 'force-repo': 'https://github.com/ceph/java_s3tests.git'}}}, {'s3tests': {'client.0': {'force-branch': 'ceph-pacific', 'rgw_server': 'client.0'}}}, {'workunit': {'clients': {'client.0': ['rgw/run-versioning.sh']}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '54e62bcbac4e53d9685e08328b790d3b20d71cae', 'timestamp': '2023-09-22_11:28:34', 'tube': 'smithi', 'user': 'cfsnyder', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.2830'} 2023-09-22T13:01:51.646 DEBUG:teuthology.orchestra.run.smithi170:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2023-09-22T13:01:51.676 DEBUG:teuthology.orchestra.run.smithi191:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2023-09-22T13:01:51.694 DEBUG:teuthology.orchestra.run.smithi170:> sudo install -d -m0777 -- /var/run/ceph 2023-09-22T13:01:51.734 DEBUG:teuthology.orchestra.run.smithi191:> sudo install -d -m0777 -- /var/run/ceph 2023-09-22T13:01:51.761 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:01:51.761 DEBUG:teuthology.orchestra.run.smithi170:> dd if=/scratch_devs of=/dev/stdout 2023-09-22T13:01:51.816 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-22T13:01:51.816 DEBUG:teuthology.orchestra.run.smithi170:> stat /dev/vg_nvme/lv_1 2023-09-22T13:01:51.874 INFO:teuthology.orchestra.run.smithi170.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-22T13:01:51.874 INFO:teuthology.orchestra.run.smithi170.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:01:51.874 INFO:teuthology.orchestra.run.smithi170.stdout:Device: 6h/6d Inode: 414142 Links: 1 2023-09-22T13:01:51.875 INFO:teuthology.orchestra.run.smithi170.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:01:51.875 INFO:teuthology.orchestra.run.smithi170.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:01:51.875 INFO:teuthology.orchestra.run.smithi170.stdout:Access: 2023-09-22 13:01:01.306747504 +0000 2023-09-22T13:01:51.875 INFO:teuthology.orchestra.run.smithi170.stdout:Modify: 2023-09-22 12:13:30.498100964 +0000 2023-09-22T13:01:51.875 INFO:teuthology.orchestra.run.smithi170.stdout:Change: 2023-09-22 12:13:30.498100964 +0000 2023-09-22T13:01:51.875 INFO:teuthology.orchestra.run.smithi170.stdout: Birth: - 2023-09-22T13:01:51.875 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-22T13:01:51.940 INFO:teuthology.orchestra.run.smithi170.stderr:1+0 records in 2023-09-22T13:01:51.940 INFO:teuthology.orchestra.run.smithi170.stderr:1+0 records out 2023-09-22T13:01:51.940 INFO:teuthology.orchestra.run.smithi170.stderr:512 bytes copied, 0.000110896 s, 4.6 MB/s 2023-09-22T13:01:51.941 DEBUG:teuthology.orchestra.run.smithi170:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-22T13:01:52.000 DEBUG:teuthology.orchestra.run.smithi170:> stat /dev/vg_nvme/lv_2 2023-09-22T13:01:52.058 INFO:teuthology.orchestra.run.smithi170.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-22T13:01:52.058 INFO:teuthology.orchestra.run.smithi170.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:01:52.058 INFO:teuthology.orchestra.run.smithi170.stdout:Device: 6h/6d Inode: 423259 Links: 1 2023-09-22T13:01:52.059 INFO:teuthology.orchestra.run.smithi170.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:01:52.059 INFO:teuthology.orchestra.run.smithi170.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:01:52.059 INFO:teuthology.orchestra.run.smithi170.stdout:Access: 2023-09-22 13:01:01.306747504 +0000 2023-09-22T13:01:52.059 INFO:teuthology.orchestra.run.smithi170.stdout:Modify: 2023-09-22 12:13:31.013088930 +0000 2023-09-22T13:01:52.059 INFO:teuthology.orchestra.run.smithi170.stdout:Change: 2023-09-22 12:13:31.013088930 +0000 2023-09-22T13:01:52.059 INFO:teuthology.orchestra.run.smithi170.stdout: Birth: - 2023-09-22T13:01:52.059 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-22T13:01:52.124 INFO:teuthology.orchestra.run.smithi170.stderr:1+0 records in 2023-09-22T13:01:52.124 INFO:teuthology.orchestra.run.smithi170.stderr:1+0 records out 2023-09-22T13:01:52.124 INFO:teuthology.orchestra.run.smithi170.stderr:512 bytes copied, 0.000107431 s, 4.8 MB/s 2023-09-22T13:01:52.125 DEBUG:teuthology.orchestra.run.smithi170:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-22T13:01:52.184 DEBUG:teuthology.orchestra.run.smithi170:> stat /dev/vg_nvme/lv_3 2023-09-22T13:01:52.242 INFO:teuthology.orchestra.run.smithi170.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-22T13:01:52.242 INFO:teuthology.orchestra.run.smithi170.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:01:52.242 INFO:teuthology.orchestra.run.smithi170.stdout:Device: 6h/6d Inode: 417384 Links: 1 2023-09-22T13:01:52.243 INFO:teuthology.orchestra.run.smithi170.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:01:52.243 INFO:teuthology.orchestra.run.smithi170.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:01:52.243 INFO:teuthology.orchestra.run.smithi170.stdout:Access: 2023-09-22 13:01:01.306747504 +0000 2023-09-22T13:01:52.243 INFO:teuthology.orchestra.run.smithi170.stdout:Modify: 2023-09-22 12:13:31.513077246 +0000 2023-09-22T13:01:52.243 INFO:teuthology.orchestra.run.smithi170.stdout:Change: 2023-09-22 12:13:31.513077246 +0000 2023-09-22T13:01:52.243 INFO:teuthology.orchestra.run.smithi170.stdout: Birth: - 2023-09-22T13:01:52.243 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-22T13:01:52.308 INFO:teuthology.orchestra.run.smithi170.stderr:1+0 records in 2023-09-22T13:01:52.308 INFO:teuthology.orchestra.run.smithi170.stderr:1+0 records out 2023-09-22T13:01:52.308 INFO:teuthology.orchestra.run.smithi170.stderr:512 bytes copied, 0.000113677 s, 4.5 MB/s 2023-09-22T13:01:52.309 DEBUG:teuthology.orchestra.run.smithi170:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-22T13:01:52.368 DEBUG:teuthology.orchestra.run.smithi170:> stat /dev/vg_nvme/lv_4 2023-09-22T13:01:52.426 INFO:teuthology.orchestra.run.smithi170.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-22T13:01:52.427 INFO:teuthology.orchestra.run.smithi170.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:01:52.427 INFO:teuthology.orchestra.run.smithi170.stdout:Device: 6h/6d Inode: 424160 Links: 1 2023-09-22T13:01:52.427 INFO:teuthology.orchestra.run.smithi170.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:01:52.427 INFO:teuthology.orchestra.run.smithi170.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:01:52.427 INFO:teuthology.orchestra.run.smithi170.stdout:Access: 2023-09-22 13:01:01.306747504 +0000 2023-09-22T13:01:52.427 INFO:teuthology.orchestra.run.smithi170.stdout:Modify: 2023-09-22 12:13:32.001065843 +0000 2023-09-22T13:01:52.427 INFO:teuthology.orchestra.run.smithi170.stdout:Change: 2023-09-22 12:13:32.001065843 +0000 2023-09-22T13:01:52.427 INFO:teuthology.orchestra.run.smithi170.stdout: Birth: - 2023-09-22T13:01:52.427 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-22T13:01:52.492 INFO:teuthology.orchestra.run.smithi170.stderr:1+0 records in 2023-09-22T13:01:52.492 INFO:teuthology.orchestra.run.smithi170.stderr:1+0 records out 2023-09-22T13:01:52.492 INFO:teuthology.orchestra.run.smithi170.stderr:512 bytes copied, 0.000123069 s, 4.2 MB/s 2023-09-22T13:01:52.493 DEBUG:teuthology.orchestra.run.smithi170:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-22T13:01:52.552 INFO:tasks.ceph:osd dev map: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:01:52.552 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:01:52.552 DEBUG:teuthology.orchestra.run.smithi191:> dd if=/scratch_devs of=/dev/stdout 2023-09-22T13:01:52.572 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-22T13:01:52.572 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_1 2023-09-22T13:01:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-22T13:01:52.632 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:01:52.632 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 6h/6d Inode: 423047 Links: 1 2023-09-22T13:01:52.632 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:01:52.632 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:01:52.632 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2023-09-22 13:01:45.049341061 +0000 2023-09-22T13:01:52.633 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2023-09-22 12:13:26.526280472 +0000 2023-09-22T13:01:52.633 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2023-09-22 12:13:26.526280472 +0000 2023-09-22T13:01:52.633 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: - 2023-09-22T13:01:52.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-22T13:01:52.700 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2023-09-22T13:01:52.700 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2023-09-22T13:01:52.701 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000274338 s, 1.9 MB/s 2023-09-22T13:01:52.702 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-22T13:01:52.763 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_2 2023-09-22T13:01:52.823 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-22T13:01:52.823 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:01:52.823 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 6h/6d Inode: 423996 Links: 1 2023-09-22T13:01:52.823 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:01:52.823 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:01:52.823 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2023-09-22 13:01:45.049341061 +0000 2023-09-22T13:01:52.824 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2023-09-22 12:13:27.036272068 +0000 2023-09-22T13:01:52.824 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2023-09-22 12:13:27.036272068 +0000 2023-09-22T13:01:52.824 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: - 2023-09-22T13:01:52.824 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-22T13:01:52.891 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2023-09-22T13:01:52.891 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2023-09-22T13:01:52.891 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000229168 s, 2.2 MB/s 2023-09-22T13:01:52.892 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-22T13:01:52.952 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_3 2023-09-22T13:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-22T13:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 6h/6d Inode: 425316 Links: 1 2023-09-22T13:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2023-09-22 13:01:45.049341061 +0000 2023-09-22T13:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2023-09-22 12:13:27.558263465 +0000 2023-09-22T13:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2023-09-22 12:13:27.558263465 +0000 2023-09-22T13:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: - 2023-09-22T13:01:53.012 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-22T13:01:53.078 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2023-09-22T13:01:53.078 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2023-09-22T13:01:53.078 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000221842 s, 2.3 MB/s 2023-09-22T13:01:53.079 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-22T13:01:53.138 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_4 2023-09-22T13:01:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-22T13:01:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:01:53.197 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 6h/6d Inode: 421721 Links: 1 2023-09-22T13:01:53.197 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:01:53.197 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:01:53.197 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2023-09-22 13:01:45.049341061 +0000 2023-09-22T13:01:53.197 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2023-09-22 12:13:28.053255307 +0000 2023-09-22T13:01:53.197 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2023-09-22 12:13:28.053255307 +0000 2023-09-22T13:01:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: - 2023-09-22T13:01:53.198 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-22T13:01:53.262 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2023-09-22T13:01:53.262 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2023-09-22T13:01:53.263 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000234127 s, 2.2 MB/s 2023-09-22T13:01:53.264 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-22T13:01:53.323 INFO:tasks.ceph:osd dev map: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:01:53.323 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi170.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi191.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2023-09-22T13:01:53.323 INFO:tasks.ceph:Generating config... 2023-09-22T13:01:53.325 INFO:tasks.ceph:[client] debug rgw = 20 2023-09-22T13:01:53.325 INFO:tasks.ceph:[client] rgw crypt require ssl = False 2023-09-22T13:01:53.325 INFO:tasks.ceph:[client] rgw crypt s3 kms backend = testing 2023-09-22T13:01:53.325 INFO:tasks.ceph:[client] rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= 2023-09-22T13:01:53.325 INFO:tasks.ceph:[client] rgw lc debug interval = 10 2023-09-22T13:01:53.325 INFO:tasks.ceph:[client] rgw max chunk size = 4194304 2023-09-22T13:01:53.325 INFO:tasks.ceph:[client] rgw obj stripe size = 6291456 2023-09-22T13:01:53.326 INFO:tasks.ceph:[client] setgroup = ceph 2023-09-22T13:01:53.326 INFO:tasks.ceph:[client] setuser = ceph 2023-09-22T13:01:53.326 INFO:tasks.ceph:[global] mon client directed command retry = 5 2023-09-22T13:01:53.326 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2023-09-22T13:01:53.326 INFO:tasks.ceph:[global] osd heartbeat grace = 40 2023-09-22T13:01:53.326 INFO:tasks.ceph:[global] osd_max_pg_log_entries = 10 2023-09-22T13:01:53.326 INFO:tasks.ceph:[global] osd_min_pg_log_entries = 10 2023-09-22T13:01:53.326 INFO:tasks.ceph:[mgr] debug mgr = 20 2023-09-22T13:01:53.327 INFO:tasks.ceph:[mgr] debug ms = 1 2023-09-22T13:01:53.327 INFO:tasks.ceph:[mon] debug mon = 20 2023-09-22T13:01:53.327 INFO:tasks.ceph:[mon] debug ms = 1 2023-09-22T13:01:53.327 INFO:tasks.ceph:[mon] debug paxos = 20 2023-09-22T13:01:53.327 INFO:tasks.ceph:[mon] mon osd crush smoke test = False 2023-09-22T13:01:53.327 INFO:tasks.ceph:[osd] bdev async discard = True 2023-09-22T13:01:53.327 INFO:tasks.ceph:[osd] bdev enable discard = True 2023-09-22T13:01:53.327 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2023-09-22T13:01:53.327 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2023-09-22T13:01:53.328 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2023-09-22T13:01:53.328 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2023-09-22T13:01:53.328 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2023-09-22T13:01:53.328 INFO:tasks.ceph:[osd] debug ms = 1 2023-09-22T13:01:53.328 INFO:tasks.ceph:[osd] debug osd = 20 2023-09-22T13:01:53.328 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2023-09-22T13:01:53.328 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2023-09-22T13:01:53.328 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2023-09-22T13:01:53.329 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2023-09-22T13:01:53.329 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2023-09-22T13:01:53.329 INFO:tasks.ceph:[osd] osd fast shutdown = False 2023-09-22T13:01:53.329 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2023-09-22T13:01:53.329 INFO:tasks.ceph:[osd] osd shutdown pgref assert = True 2023-09-22T13:01:53.329 INFO:tasks.ceph:Setting up mon.a... 2023-09-22T13:01:53.329 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring 2023-09-22T13:01:53.445 INFO:teuthology.orchestra.run.smithi170.stdout:creating /etc/ceph/ceph.keyring 2023-09-22T13:01:53.447 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring 2023-09-22T13:01:53.494 DEBUG:teuthology.orchestra.run.smithi170:> sudo chmod 0644 /etc/ceph/ceph.keyring 2023-09-22T13:01:53.523 DEBUG:tasks.ceph:Ceph mon addresses: [('mon.a', '172.21.15.170'), ('mon.c', '[v2:172.21.15.170:3301,v1:172.21.15.170:6790]'), ('mon.b', '172.21.15.191')] 2023-09-22T13:01:53.523 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.170,[v2:172.21.15.170:3301,v1:172.21.15.170:6790],172.21.15.191', 'mon client directed command retry': 5, 'ms inject socket failures': 5000, 'osd heartbeat grace': 40, 'osd_max_pg_log_entries': 10, 'osd_min_pg_log_entries': 10}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd fast shutdown': False, 'osd objectstore': 'bluestore', 'osd shutdown pgref assert': True}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon osd crush smoke test': False}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=', 'rgw lc debug interval': 10, 'rgw max chunk size': 4194304, 'rgw obj stripe size': 6291456, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mon.a': {}, 'mon.c': {}, 'mon.b': {}} 2023-09-22T13:01:53.524 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:01:53.524 DEBUG:teuthology.orchestra.run.smithi170:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2023-09-22T13:01:53.582 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.170 --addv c '[v2:172.21.15.170:3301,v1:172.21.15.170:6790]' --add b 172.21.15.191 --print /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:01:53.667 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:01:53.667 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:01:53.668 INFO:teuthology.orchestra.run.smithi170.stdout:monmaptool: monmap file /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:01:53.668 INFO:teuthology.orchestra.run.smithi170.stdout:monmaptool: generated fsid 3f2e844c-1843-4fab-beef-461ea345cf51 2023-09-22T13:01:53.668 INFO:teuthology.orchestra.run.smithi170.stdout:epoch 0 2023-09-22T13:01:53.668 INFO:teuthology.orchestra.run.smithi170.stdout:fsid 3f2e844c-1843-4fab-beef-461ea345cf51 2023-09-22T13:01:53.669 INFO:teuthology.orchestra.run.smithi170.stdout:last_changed 2023-09-22T13:01:53.668404+0000 2023-09-22T13:01:53.669 INFO:teuthology.orchestra.run.smithi170.stdout:created 2023-09-22T13:01:53.668404+0000 2023-09-22T13:01:53.669 INFO:teuthology.orchestra.run.smithi170.stdout:min_mon_release 0 (unknown) 2023-09-22T13:01:53.669 INFO:teuthology.orchestra.run.smithi170.stdout:election_strategy: 1 2023-09-22T13:01:53.669 INFO:teuthology.orchestra.run.smithi170.stdout:0: [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] mon.a 2023-09-22T13:01:53.669 INFO:teuthology.orchestra.run.smithi170.stdout:1: [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon.b 2023-09-22T13:01:53.669 INFO:teuthology.orchestra.run.smithi170.stdout:2: [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] mon.c 2023-09-22T13:01:53.669 INFO:teuthology.orchestra.run.smithi170.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/ceph.monmap (3 monitors) 2023-09-22T13:01:53.671 DEBUG:teuthology.orchestra.run.smithi170:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2023-09-22T13:01:53.689 INFO:tasks.ceph:Writing /etc/ceph/ceph.conf for FSID 3f2e844c-1843-4fab-beef-461ea345cf51... 2023-09-22T13:01:53.690 DEBUG:teuthology.orchestra.run.smithi170:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2023-09-22T13:01:53.732 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2023-09-22T13:01:53.775 INFO:teuthology.orchestra.run.smithi170.stdout:[global] 2023-09-22T13:01:53.775 INFO:teuthology.orchestra.run.smithi170.stdout: chdir = "" 2023-09-22T13:01:53.775 INFO:teuthology.orchestra.run.smithi170.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2023-09-22T13:01:53.775 INFO:teuthology.orchestra.run.smithi170.stdout: auth supported = cephx 2023-09-22T13:01:53.775 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.775 INFO:teuthology.orchestra.run.smithi170.stdout: filestore xattr use omap = true 2023-09-22T13:01:53.775 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.776 INFO:teuthology.orchestra.run.smithi170.stdout: mon clock drift allowed = 1.000 2023-09-22T13:01:53.776 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.776 INFO:teuthology.orchestra.run.smithi170.stdout: osd crush chooseleaf type = 0 2023-09-22T13:01:53.776 INFO:teuthology.orchestra.run.smithi170.stdout: auth debug = true 2023-09-22T13:01:53.776 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.776 INFO:teuthology.orchestra.run.smithi170.stdout: ms die on old message = true 2023-09-22T13:01:53.776 INFO:teuthology.orchestra.run.smithi170.stdout: ms die on bug = true 2023-09-22T13:01:53.776 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.777 INFO:teuthology.orchestra.run.smithi170.stdout: mon max pg per osd = 10000 # >= luminous 2023-09-22T13:01:53.777 INFO:teuthology.orchestra.run.smithi170.stdout: mon pg warn max object skew = 0 2023-09-22T13:01:53.777 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.777 INFO:teuthology.orchestra.run.smithi170.stdout: # disable pg_autoscaler by default for new pools 2023-09-22T13:01:53.777 INFO:teuthology.orchestra.run.smithi170.stdout: osd_pool_default_pg_autoscale_mode = off 2023-09-22T13:01:53.777 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.777 INFO:teuthology.orchestra.run.smithi170.stdout: osd pool default size = 2 2023-09-22T13:01:53.777 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.778 INFO:teuthology.orchestra.run.smithi170.stdout: mon osd allow primary affinity = true 2023-09-22T13:01:53.778 INFO:teuthology.orchestra.run.smithi170.stdout: mon osd allow pg remap = true 2023-09-22T13:01:53.778 INFO:teuthology.orchestra.run.smithi170.stdout: mon warn on legacy crush tunables = false 2023-09-22T13:01:53.778 INFO:teuthology.orchestra.run.smithi170.stdout: mon warn on crush straw calc version zero = false 2023-09-22T13:01:53.778 INFO:teuthology.orchestra.run.smithi170.stdout: mon warn on no sortbitwise = false 2023-09-22T13:01:53.778 INFO:teuthology.orchestra.run.smithi170.stdout: mon warn on osd down out interval zero = false 2023-09-22T13:01:53.778 INFO:teuthology.orchestra.run.smithi170.stdout: mon warn on too few osds = false 2023-09-22T13:01:53.778 INFO:teuthology.orchestra.run.smithi170.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2023-09-22T13:01:53.779 INFO:teuthology.orchestra.run.smithi170.stdout: mon_warn_on_pool_no_redundancy = false 2023-09-22T13:01:53.779 INFO:teuthology.orchestra.run.smithi170.stdout: mon_allow_pool_size_one = true 2023-09-22T13:01:53.779 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.779 INFO:teuthology.orchestra.run.smithi170.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd 2023-09-22T13:01:53.779 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.779 INFO:teuthology.orchestra.run.smithi170.stdout: osd default data pool replay window = 5 2023-09-22T13:01:53.779 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.779 INFO:teuthology.orchestra.run.smithi170.stdout: mon allow pool delete = true 2023-09-22T13:01:53.780 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.780 INFO:teuthology.orchestra.run.smithi170.stdout: mon cluster log file level = debug 2023-09-22T13:01:53.780 INFO:teuthology.orchestra.run.smithi170.stdout: debug asserts on shutdown = true 2023-09-22T13:01:53.780 INFO:teuthology.orchestra.run.smithi170.stdout: mon health detail to clog = false 2023-09-22T13:01:53.780 INFO:teuthology.orchestra.run.smithi170.stdout: mon host = "172.21.15.170,[v2:172.21.15.170:3301,v1:172.21.15.170:6790],172.21.15.191" 2023-09-22T13:01:53.780 INFO:teuthology.orchestra.run.smithi170.stdout: mon client directed command retry = 5 2023-09-22T13:01:53.780 INFO:teuthology.orchestra.run.smithi170.stdout: ms inject socket failures = 5000 2023-09-22T13:01:53.780 INFO:teuthology.orchestra.run.smithi170.stdout: osd heartbeat grace = 40 2023-09-22T13:01:53.781 INFO:teuthology.orchestra.run.smithi170.stdout: osd_max_pg_log_entries = 10 2023-09-22T13:01:53.781 INFO:teuthology.orchestra.run.smithi170.stdout: osd_min_pg_log_entries = 10 2023-09-22T13:01:53.781 INFO:teuthology.orchestra.run.smithi170.stdout: fsid = 3f2e844c-1843-4fab-beef-461ea345cf51 2023-09-22T13:01:53.781 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.781 INFO:teuthology.orchestra.run.smithi170.stdout:[osd] 2023-09-22T13:01:53.781 INFO:teuthology.orchestra.run.smithi170.stdout: osd journal size = 100 2023-09-22T13:01:53.781 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.781 INFO:teuthology.orchestra.run.smithi170.stdout: osd scrub load threshold = 5.0 2023-09-22T13:01:53.782 INFO:teuthology.orchestra.run.smithi170.stdout: osd scrub max interval = 600 2023-09-22T13:01:53.782 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.782 INFO:teuthology.orchestra.run.smithi170.stdout: osd recover clone overlap = true 2023-09-22T13:01:53.782 INFO:teuthology.orchestra.run.smithi170.stdout: osd recovery max chunk = 1048576 2023-09-22T13:01:53.782 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.782 INFO:teuthology.orchestra.run.smithi170.stdout: osd debug shutdown = true 2023-09-22T13:01:53.782 INFO:teuthology.orchestra.run.smithi170.stdout: osd debug op order = true 2023-09-22T13:01:53.783 INFO:teuthology.orchestra.run.smithi170.stdout: osd debug verify stray on activate = true 2023-09-22T13:01:53.783 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.783 INFO:teuthology.orchestra.run.smithi170.stdout: osd open classes on start = true 2023-09-22T13:01:53.783 INFO:teuthology.orchestra.run.smithi170.stdout: osd debug pg log writeout = true 2023-09-22T13:01:53.783 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.783 INFO:teuthology.orchestra.run.smithi170.stdout: osd deep scrub update digest min age = 30 2023-09-22T13:01:53.783 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.783 INFO:teuthology.orchestra.run.smithi170.stdout: osd map max advance = 10 2023-09-22T13:01:53.784 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.784 INFO:teuthology.orchestra.run.smithi170.stdout: journal zero on create = true 2023-09-22T13:01:53.784 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.784 INFO:teuthology.orchestra.run.smithi170.stdout: filestore ondisk finisher threads = 3 2023-09-22T13:01:53.784 INFO:teuthology.orchestra.run.smithi170.stdout: filestore apply finisher threads = 3 2023-09-22T13:01:53.784 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.784 INFO:teuthology.orchestra.run.smithi170.stdout: bdev debug aio = true 2023-09-22T13:01:53.784 INFO:teuthology.orchestra.run.smithi170.stdout: osd debug misdirected ops = true 2023-09-22T13:01:53.785 INFO:teuthology.orchestra.run.smithi170.stdout: bdev async discard = True 2023-09-22T13:01:53.785 INFO:teuthology.orchestra.run.smithi170.stdout: bdev enable discard = True 2023-09-22T13:01:53.785 INFO:teuthology.orchestra.run.smithi170.stdout: bluestore allocator = bitmap 2023-09-22T13:01:53.785 INFO:teuthology.orchestra.run.smithi170.stdout: bluestore block size = 96636764160 2023-09-22T13:01:53.785 INFO:teuthology.orchestra.run.smithi170.stdout: bluestore fsck on mount = True 2023-09-22T13:01:53.785 INFO:teuthology.orchestra.run.smithi170.stdout: debug bluefs = 1/20 2023-09-22T13:01:53.785 INFO:teuthology.orchestra.run.smithi170.stdout: debug bluestore = 1/20 2023-09-22T13:01:53.785 INFO:teuthology.orchestra.run.smithi170.stdout: debug ms = 1 2023-09-22T13:01:53.786 INFO:teuthology.orchestra.run.smithi170.stdout: debug osd = 20 2023-09-22T13:01:53.786 INFO:teuthology.orchestra.run.smithi170.stdout: debug rocksdb = 4/10 2023-09-22T13:01:53.786 INFO:teuthology.orchestra.run.smithi170.stdout: mon osd backfillfull_ratio = 0.85 2023-09-22T13:01:53.786 INFO:teuthology.orchestra.run.smithi170.stdout: mon osd full ratio = 0.9 2023-09-22T13:01:53.786 INFO:teuthology.orchestra.run.smithi170.stdout: mon osd nearfull ratio = 0.8 2023-09-22T13:01:53.786 INFO:teuthology.orchestra.run.smithi170.stdout: osd failsafe full ratio = 0.95 2023-09-22T13:01:53.786 INFO:teuthology.orchestra.run.smithi170.stdout: osd fast shutdown = False 2023-09-22T13:01:53.786 INFO:teuthology.orchestra.run.smithi170.stdout: osd objectstore = bluestore 2023-09-22T13:01:53.787 INFO:teuthology.orchestra.run.smithi170.stdout: osd shutdown pgref assert = True 2023-09-22T13:01:53.787 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.787 INFO:teuthology.orchestra.run.smithi170.stdout:[mgr] 2023-09-22T13:01:53.787 INFO:teuthology.orchestra.run.smithi170.stdout: debug ms = 1 2023-09-22T13:01:53.787 INFO:teuthology.orchestra.run.smithi170.stdout: debug mgr = 20 2023-09-22T13:01:53.787 INFO:teuthology.orchestra.run.smithi170.stdout: debug mon = 20 2023-09-22T13:01:53.787 INFO:teuthology.orchestra.run.smithi170.stdout: debug auth = 20 2023-09-22T13:01:53.787 INFO:teuthology.orchestra.run.smithi170.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:01:53.788 INFO:teuthology.orchestra.run.smithi170.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:01:53.788 INFO:teuthology.orchestra.run.smithi170.stdout: mgr/telemetry/nag = false 2023-09-22T13:01:53.788 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.788 INFO:teuthology.orchestra.run.smithi170.stdout:[mon] 2023-09-22T13:01:53.788 INFO:teuthology.orchestra.run.smithi170.stdout: debug ms = 1 2023-09-22T13:01:53.788 INFO:teuthology.orchestra.run.smithi170.stdout: debug mon = 20 2023-09-22T13:01:53.788 INFO:teuthology.orchestra.run.smithi170.stdout: debug paxos = 20 2023-09-22T13:01:53.788 INFO:teuthology.orchestra.run.smithi170.stdout: debug auth = 20 2023-09-22T13:01:53.789 INFO:teuthology.orchestra.run.smithi170.stdout: mon data avail warn = 5 2023-09-22T13:01:53.789 INFO:teuthology.orchestra.run.smithi170.stdout: mon mgr mkfs grace = 240 2023-09-22T13:01:53.789 INFO:teuthology.orchestra.run.smithi170.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:01:53.789 INFO:teuthology.orchestra.run.smithi170.stdout: mon osd reporter subtree level = osd 2023-09-22T13:01:53.789 INFO:teuthology.orchestra.run.smithi170.stdout: mon osd prime pg temp = true 2023-09-22T13:01:53.789 INFO:teuthology.orchestra.run.smithi170.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:01:53.789 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.789 INFO:teuthology.orchestra.run.smithi170.stdout: # rotate auth tickets quickly to exercise renewal paths 2023-09-22T13:01:53.789 INFO:teuthology.orchestra.run.smithi170.stdout: auth mon ticket ttl = 660 # 11m 2023-09-22T13:01:53.790 INFO:teuthology.orchestra.run.smithi170.stdout: auth service ticket ttl = 240 # 4m 2023-09-22T13:01:53.790 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.790 INFO:teuthology.orchestra.run.smithi170.stdout: # don't complain about insecure global_id in the test suite 2023-09-22T13:01:53.790 INFO:teuthology.orchestra.run.smithi170.stdout: mon_warn_on_insecure_global_id_reclaim = false 2023-09-22T13:01:53.790 INFO:teuthology.orchestra.run.smithi170.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2023-09-22T13:01:53.790 INFO:teuthology.orchestra.run.smithi170.stdout: mon osd crush smoke test = False 2023-09-22T13:01:53.790 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:01:53.791 INFO:teuthology.orchestra.run.smithi170.stdout:[client] 2023-09-22T13:01:53.791 INFO:teuthology.orchestra.run.smithi170.stdout: rgw cache enabled = true 2023-09-22T13:01:53.791 INFO:teuthology.orchestra.run.smithi170.stdout: rgw enable ops log = true 2023-09-22T13:01:53.791 INFO:teuthology.orchestra.run.smithi170.stdout: rgw enable usage log = true 2023-09-22T13:01:53.791 INFO:teuthology.orchestra.run.smithi170.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2023-09-22T13:01:53.791 INFO:teuthology.orchestra.run.smithi170.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2023-09-22T13:01:53.791 INFO:teuthology.orchestra.run.smithi170.stdout: debug rgw = 20 2023-09-22T13:01:53.791 INFO:teuthology.orchestra.run.smithi170.stdout: rgw crypt require ssl = False 2023-09-22T13:01:53.792 INFO:teuthology.orchestra.run.smithi170.stdout: rgw crypt s3 kms backend = testing 2023-09-22T13:01:53.792 INFO:teuthology.orchestra.run.smithi170.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= 2023-09-22T13:01:53.792 INFO:teuthology.orchestra.run.smithi170.stdout: rgw lc debug interval = 10 2023-09-22T13:01:53.792 INFO:teuthology.orchestra.run.smithi170.stdout: rgw max chunk size = 4194304 2023-09-22T13:01:53.792 INFO:teuthology.orchestra.run.smithi170.stdout: rgw obj stripe size = 6291456 2023-09-22T13:01:53.792 INFO:teuthology.orchestra.run.smithi170.stdout: setgroup = ceph 2023-09-22T13:01:53.792 INFO:teuthology.orchestra.run.smithi170.stdout: setuser = ceph 2023-09-22T13:01:53.792 INFO:teuthology.orchestra.run.smithi170.stdout:[mon.a] 2023-09-22T13:01:53.792 INFO:teuthology.orchestra.run.smithi170.stdout:[mon.c] 2023-09-22T13:01:53.793 INFO:teuthology.orchestra.run.smithi170.stdout:[mon.b] 2023-09-22T13:01:53.794 INFO:teuthology.orchestra.run.smithi191.stdout:[global] 2023-09-22T13:01:53.794 INFO:teuthology.orchestra.run.smithi191.stdout: chdir = "" 2023-09-22T13:01:53.795 INFO:teuthology.orchestra.run.smithi191.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2023-09-22T13:01:53.795 INFO:teuthology.orchestra.run.smithi191.stdout: auth supported = cephx 2023-09-22T13:01:53.795 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.795 INFO:teuthology.orchestra.run.smithi191.stdout: filestore xattr use omap = true 2023-09-22T13:01:53.795 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.795 INFO:teuthology.orchestra.run.smithi191.stdout: mon clock drift allowed = 1.000 2023-09-22T13:01:53.795 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.795 INFO:teuthology.orchestra.run.smithi191.stdout: osd crush chooseleaf type = 0 2023-09-22T13:01:53.796 INFO:teuthology.orchestra.run.smithi191.stdout: auth debug = true 2023-09-22T13:01:53.796 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.796 INFO:teuthology.orchestra.run.smithi191.stdout: ms die on old message = true 2023-09-22T13:01:53.796 INFO:teuthology.orchestra.run.smithi191.stdout: ms die on bug = true 2023-09-22T13:01:53.796 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.796 INFO:teuthology.orchestra.run.smithi191.stdout: mon max pg per osd = 10000 # >= luminous 2023-09-22T13:01:53.796 INFO:teuthology.orchestra.run.smithi191.stdout: mon pg warn max object skew = 0 2023-09-22T13:01:53.796 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.796 INFO:teuthology.orchestra.run.smithi191.stdout: # disable pg_autoscaler by default for new pools 2023-09-22T13:01:53.797 INFO:teuthology.orchestra.run.smithi191.stdout: osd_pool_default_pg_autoscale_mode = off 2023-09-22T13:01:53.797 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.797 INFO:teuthology.orchestra.run.smithi191.stdout: osd pool default size = 2 2023-09-22T13:01:53.797 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.797 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd allow primary affinity = true 2023-09-22T13:01:53.797 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd allow pg remap = true 2023-09-22T13:01:53.797 INFO:teuthology.orchestra.run.smithi191.stdout: mon warn on legacy crush tunables = false 2023-09-22T13:01:53.797 INFO:teuthology.orchestra.run.smithi191.stdout: mon warn on crush straw calc version zero = false 2023-09-22T13:01:53.797 INFO:teuthology.orchestra.run.smithi191.stdout: mon warn on no sortbitwise = false 2023-09-22T13:01:53.798 INFO:teuthology.orchestra.run.smithi191.stdout: mon warn on osd down out interval zero = false 2023-09-22T13:01:53.798 INFO:teuthology.orchestra.run.smithi191.stdout: mon warn on too few osds = false 2023-09-22T13:01:53.798 INFO:teuthology.orchestra.run.smithi191.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2023-09-22T13:01:53.798 INFO:teuthology.orchestra.run.smithi191.stdout: mon_warn_on_pool_no_redundancy = false 2023-09-22T13:01:53.798 INFO:teuthology.orchestra.run.smithi191.stdout: mon_allow_pool_size_one = true 2023-09-22T13:01:53.798 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.798 INFO:teuthology.orchestra.run.smithi191.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd 2023-09-22T13:01:53.798 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.798 INFO:teuthology.orchestra.run.smithi191.stdout: osd default data pool replay window = 5 2023-09-22T13:01:53.799 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.799 INFO:teuthology.orchestra.run.smithi191.stdout: mon allow pool delete = true 2023-09-22T13:01:53.799 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.799 INFO:teuthology.orchestra.run.smithi191.stdout: mon cluster log file level = debug 2023-09-22T13:01:53.799 INFO:teuthology.orchestra.run.smithi191.stdout: debug asserts on shutdown = true 2023-09-22T13:01:53.799 INFO:teuthology.orchestra.run.smithi191.stdout: mon health detail to clog = false 2023-09-22T13:01:53.799 INFO:teuthology.orchestra.run.smithi191.stdout: mon host = "172.21.15.170,[v2:172.21.15.170:3301,v1:172.21.15.170:6790],172.21.15.191" 2023-09-22T13:01:53.799 INFO:teuthology.orchestra.run.smithi191.stdout: mon client directed command retry = 5 2023-09-22T13:01:53.799 INFO:teuthology.orchestra.run.smithi191.stdout: ms inject socket failures = 5000 2023-09-22T13:01:53.799 INFO:teuthology.orchestra.run.smithi191.stdout: osd heartbeat grace = 40 2023-09-22T13:01:53.800 INFO:teuthology.orchestra.run.smithi191.stdout: osd_max_pg_log_entries = 10 2023-09-22T13:01:53.800 INFO:teuthology.orchestra.run.smithi191.stdout: osd_min_pg_log_entries = 10 2023-09-22T13:01:53.800 INFO:teuthology.orchestra.run.smithi191.stdout: fsid = 3f2e844c-1843-4fab-beef-461ea345cf51 2023-09-22T13:01:53.800 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.800 INFO:teuthology.orchestra.run.smithi191.stdout:[osd] 2023-09-22T13:01:53.800 INFO:teuthology.orchestra.run.smithi191.stdout: osd journal size = 100 2023-09-22T13:01:53.800 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.800 INFO:teuthology.orchestra.run.smithi191.stdout: osd scrub load threshold = 5.0 2023-09-22T13:01:53.800 INFO:teuthology.orchestra.run.smithi191.stdout: osd scrub max interval = 600 2023-09-22T13:01:53.801 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.801 INFO:teuthology.orchestra.run.smithi191.stdout: osd recover clone overlap = true 2023-09-22T13:01:53.801 INFO:teuthology.orchestra.run.smithi191.stdout: osd recovery max chunk = 1048576 2023-09-22T13:01:53.801 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.801 INFO:teuthology.orchestra.run.smithi191.stdout: osd debug shutdown = true 2023-09-22T13:01:53.801 INFO:teuthology.orchestra.run.smithi191.stdout: osd debug op order = true 2023-09-22T13:01:53.801 INFO:teuthology.orchestra.run.smithi191.stdout: osd debug verify stray on activate = true 2023-09-22T13:01:53.801 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.801 INFO:teuthology.orchestra.run.smithi191.stdout: osd open classes on start = true 2023-09-22T13:01:53.801 INFO:teuthology.orchestra.run.smithi191.stdout: osd debug pg log writeout = true 2023-09-22T13:01:53.802 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.802 INFO:teuthology.orchestra.run.smithi191.stdout: osd deep scrub update digest min age = 30 2023-09-22T13:01:53.802 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.802 INFO:teuthology.orchestra.run.smithi191.stdout: osd map max advance = 10 2023-09-22T13:01:53.802 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.802 INFO:teuthology.orchestra.run.smithi191.stdout: journal zero on create = true 2023-09-22T13:01:53.802 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.802 INFO:teuthology.orchestra.run.smithi191.stdout: filestore ondisk finisher threads = 3 2023-09-22T13:01:53.802 INFO:teuthology.orchestra.run.smithi191.stdout: filestore apply finisher threads = 3 2023-09-22T13:01:53.803 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.803 INFO:teuthology.orchestra.run.smithi191.stdout: bdev debug aio = true 2023-09-22T13:01:53.803 INFO:teuthology.orchestra.run.smithi191.stdout: osd debug misdirected ops = true 2023-09-22T13:01:53.803 INFO:teuthology.orchestra.run.smithi191.stdout: bdev async discard = True 2023-09-22T13:01:53.803 INFO:teuthology.orchestra.run.smithi191.stdout: bdev enable discard = True 2023-09-22T13:01:53.803 INFO:teuthology.orchestra.run.smithi191.stdout: bluestore allocator = bitmap 2023-09-22T13:01:53.803 INFO:teuthology.orchestra.run.smithi191.stdout: bluestore block size = 96636764160 2023-09-22T13:01:53.803 INFO:teuthology.orchestra.run.smithi191.stdout: bluestore fsck on mount = True 2023-09-22T13:01:53.803 INFO:teuthology.orchestra.run.smithi191.stdout: debug bluefs = 1/20 2023-09-22T13:01:53.804 INFO:teuthology.orchestra.run.smithi191.stdout: debug bluestore = 1/20 2023-09-22T13:01:53.804 INFO:teuthology.orchestra.run.smithi191.stdout: debug ms = 1 2023-09-22T13:01:53.804 INFO:teuthology.orchestra.run.smithi191.stdout: debug osd = 20 2023-09-22T13:01:53.804 INFO:teuthology.orchestra.run.smithi191.stdout: debug rocksdb = 4/10 2023-09-22T13:01:53.804 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd backfillfull_ratio = 0.85 2023-09-22T13:01:53.804 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd full ratio = 0.9 2023-09-22T13:01:53.804 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd nearfull ratio = 0.8 2023-09-22T13:01:53.804 INFO:teuthology.orchestra.run.smithi191.stdout: osd failsafe full ratio = 0.95 2023-09-22T13:01:53.804 INFO:teuthology.orchestra.run.smithi191.stdout: osd fast shutdown = False 2023-09-22T13:01:53.805 INFO:teuthology.orchestra.run.smithi191.stdout: osd objectstore = bluestore 2023-09-22T13:01:53.805 INFO:teuthology.orchestra.run.smithi191.stdout: osd shutdown pgref assert = True 2023-09-22T13:01:53.805 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.805 INFO:teuthology.orchestra.run.smithi191.stdout:[mgr] 2023-09-22T13:01:53.805 INFO:teuthology.orchestra.run.smithi191.stdout: debug ms = 1 2023-09-22T13:01:53.805 INFO:teuthology.orchestra.run.smithi191.stdout: debug mgr = 20 2023-09-22T13:01:53.805 INFO:teuthology.orchestra.run.smithi191.stdout: debug mon = 20 2023-09-22T13:01:53.805 INFO:teuthology.orchestra.run.smithi191.stdout: debug auth = 20 2023-09-22T13:01:53.805 INFO:teuthology.orchestra.run.smithi191.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:01:53.806 INFO:teuthology.orchestra.run.smithi191.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:01:53.806 INFO:teuthology.orchestra.run.smithi191.stdout: mgr/telemetry/nag = false 2023-09-22T13:01:53.806 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.806 INFO:teuthology.orchestra.run.smithi191.stdout:[mon] 2023-09-22T13:01:53.806 INFO:teuthology.orchestra.run.smithi191.stdout: debug ms = 1 2023-09-22T13:01:53.806 INFO:teuthology.orchestra.run.smithi191.stdout: debug mon = 20 2023-09-22T13:01:53.806 INFO:teuthology.orchestra.run.smithi191.stdout: debug paxos = 20 2023-09-22T13:01:53.806 INFO:teuthology.orchestra.run.smithi191.stdout: debug auth = 20 2023-09-22T13:01:53.806 INFO:teuthology.orchestra.run.smithi191.stdout: mon data avail warn = 5 2023-09-22T13:01:53.806 INFO:teuthology.orchestra.run.smithi191.stdout: mon mgr mkfs grace = 240 2023-09-22T13:01:53.807 INFO:teuthology.orchestra.run.smithi191.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:01:53.807 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd reporter subtree level = osd 2023-09-22T13:01:53.807 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd prime pg temp = true 2023-09-22T13:01:53.807 INFO:teuthology.orchestra.run.smithi191.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:01:53.807 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.807 INFO:teuthology.orchestra.run.smithi191.stdout: # rotate auth tickets quickly to exercise renewal paths 2023-09-22T13:01:53.807 INFO:teuthology.orchestra.run.smithi191.stdout: auth mon ticket ttl = 660 # 11m 2023-09-22T13:01:53.807 INFO:teuthology.orchestra.run.smithi191.stdout: auth service ticket ttl = 240 # 4m 2023-09-22T13:01:53.807 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.807 INFO:teuthology.orchestra.run.smithi191.stdout: # don't complain about insecure global_id in the test suite 2023-09-22T13:01:53.808 INFO:teuthology.orchestra.run.smithi191.stdout: mon_warn_on_insecure_global_id_reclaim = false 2023-09-22T13:01:53.808 INFO:teuthology.orchestra.run.smithi191.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2023-09-22T13:01:53.808 INFO:teuthology.orchestra.run.smithi191.stdout: mon osd crush smoke test = False 2023-09-22T13:01:53.808 INFO:teuthology.orchestra.run.smithi191.stdout: 2023-09-22T13:01:53.808 INFO:teuthology.orchestra.run.smithi191.stdout:[client] 2023-09-22T13:01:53.808 INFO:teuthology.orchestra.run.smithi191.stdout: rgw cache enabled = true 2023-09-22T13:01:53.808 INFO:teuthology.orchestra.run.smithi191.stdout: rgw enable ops log = true 2023-09-22T13:01:53.808 INFO:teuthology.orchestra.run.smithi191.stdout: rgw enable usage log = true 2023-09-22T13:01:53.808 INFO:teuthology.orchestra.run.smithi191.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2023-09-22T13:01:53.808 INFO:teuthology.orchestra.run.smithi191.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2023-09-22T13:01:53.809 INFO:teuthology.orchestra.run.smithi191.stdout: debug rgw = 20 2023-09-22T13:01:53.809 INFO:teuthology.orchestra.run.smithi191.stdout: rgw crypt require ssl = False 2023-09-22T13:01:53.809 INFO:teuthology.orchestra.run.smithi191.stdout: rgw crypt s3 kms backend = testing 2023-09-22T13:01:53.809 INFO:teuthology.orchestra.run.smithi191.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= 2023-09-22T13:01:53.809 INFO:teuthology.orchestra.run.smithi191.stdout: rgw lc debug interval = 10 2023-09-22T13:01:53.809 INFO:teuthology.orchestra.run.smithi191.stdout: rgw max chunk size = 4194304 2023-09-22T13:01:53.809 INFO:teuthology.orchestra.run.smithi191.stdout: rgw obj stripe size = 6291456 2023-09-22T13:01:53.809 INFO:teuthology.orchestra.run.smithi191.stdout: setgroup = ceph 2023-09-22T13:01:53.809 INFO:teuthology.orchestra.run.smithi191.stdout: setuser = ceph 2023-09-22T13:01:53.809 INFO:teuthology.orchestra.run.smithi191.stdout:[mon.a] 2023-09-22T13:01:53.810 INFO:teuthology.orchestra.run.smithi191.stdout:[mon.c] 2023-09-22T13:01:53.810 INFO:teuthology.orchestra.run.smithi191.stdout:[mon.b] 2023-09-22T13:01:53.810 INFO:tasks.ceph:Creating admin key on mon.a... 2023-09-22T13:01:53.810 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/ceph.keyring 2023-09-22T13:01:53.887 INFO:tasks.ceph:Copying monmap to all nodes... 2023-09-22T13:01:53.887 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:01:53.887 DEBUG:teuthology.orchestra.run.smithi170:> dd if=/etc/ceph/ceph.keyring of=/dev/stdout 2023-09-22T13:01:53.904 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:01:53.905 DEBUG:teuthology.orchestra.run.smithi170:> dd if=/home/ubuntu/cephtest/ceph.monmap of=/dev/stdout 2023-09-22T13:01:53.962 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi170.front.sepia.ceph.com 2023-09-22T13:01:53.963 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:01:53.963 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd of=/etc/ceph/ceph.keyring 2023-09-22T13:01:53.963 DEBUG:teuthology.orchestra.run.smithi170:> sudo chmod 0644 /etc/ceph/ceph.keyring 2023-09-22T13:01:54.039 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:01:54.039 DEBUG:teuthology.orchestra.run.smithi170:> dd of=/home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:01:54.098 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi191.front.sepia.ceph.com 2023-09-22T13:01:54.098 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:01:54.098 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/ceph/ceph.keyring 2023-09-22T13:01:54.098 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod 0644 /etc/ceph/ceph.keyring 2023-09-22T13:01:54.137 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:01:54.137 DEBUG:teuthology.orchestra.run.smithi191:> dd of=/home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:01:54.196 INFO:tasks.ceph:Setting up mon nodes... 2023-09-22T13:01:54.196 INFO:tasks.ceph:Setting up mgr nodes... 2023-09-22T13:01:54.197 DEBUG:teuthology.orchestra.run.smithi170:> sudo mkdir -p /var/lib/ceph/mgr/ceph-y && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.y /var/lib/ceph/mgr/ceph-y/keyring 2023-09-22T13:01:54.246 INFO:teuthology.orchestra.run.smithi170.stdout:creating /var/lib/ceph/mgr/ceph-y/keyring 2023-09-22T13:01:54.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/mgr/ceph-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/ceph-x/keyring 2023-09-22T13:01:54.361 INFO:teuthology.orchestra.run.smithi191.stdout:creating /var/lib/ceph/mgr/ceph-x/keyring 2023-09-22T13:01:54.364 INFO:tasks.ceph:Setting up mds nodes... 2023-09-22T13:01:54.364 INFO:tasks.ceph_client:Setting up client nodes... 2023-09-22T13:01:54.364 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/ceph.client.0.keyring && sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-22T13:01:54.404 INFO:teuthology.orchestra.run.smithi170.stdout:creating /etc/ceph/ceph.client.0.keyring 2023-09-22T13:01:54.416 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/ceph.client.1.keyring && sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-22T13:01:54.457 INFO:teuthology.orchestra.run.smithi191.stdout:creating /etc/ceph/ceph.client.1.keyring 2023-09-22T13:01:54.469 INFO:tasks.ceph:Running mkfs on osd nodes... 2023-09-22T13:01:54.469 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi170.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi191.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2023-09-22T13:01:54.470 DEBUG:teuthology.orchestra.run.smithi170:> sudo mkdir -p /var/lib/ceph/osd/ceph-0 2023-09-22T13:01:54.497 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:01:54.497 INFO:tasks.ceph:role: osd.0 2023-09-22T13:01:54.497 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi170.front.sepia.ceph.com 2023-09-22T13:01:54.497 DEBUG:teuthology.orchestra.run.smithi170:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2023-09-22T13:01:54.564 INFO:teuthology.orchestra.run.smithi170.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:01:54.565 INFO:teuthology.orchestra.run.smithi170.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:01:54.565 INFO:teuthology.orchestra.run.smithi170.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:01:54.565 INFO:teuthology.orchestra.run.smithi170.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:01:54.565 INFO:teuthology.orchestra.run.smithi170.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:01:54.565 INFO:teuthology.orchestra.run.smithi170.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:01:54.565 INFO:teuthology.orchestra.run.smithi170.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:01:54.565 INFO:teuthology.orchestra.run.smithi170.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:01:54.566 INFO:teuthology.orchestra.run.smithi170.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:01:54.566 INFO:teuthology.orchestra.run.smithi170.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:01:54.587 INFO:teuthology.orchestra.run.smithi170.stdout:Discarding blocks...Done. 2023-09-22T13:01:54.589 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi170.front.sepia.ceph.com -o noatime 2023-09-22T13:01:54.589 DEBUG:teuthology.orchestra.run.smithi170:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-0 2023-09-22T13:01:54.671 DEBUG:teuthology.orchestra.run.smithi170:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-0 2023-09-22T13:01:54.744 DEBUG:teuthology.orchestra.run.smithi170:> sudo mkdir -p /var/lib/ceph/osd/ceph-1 2023-09-22T13:01:54.811 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:01:54.811 INFO:tasks.ceph:role: osd.1 2023-09-22T13:01:54.811 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi170.front.sepia.ceph.com 2023-09-22T13:01:54.811 DEBUG:teuthology.orchestra.run.smithi170:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2023-09-22T13:01:54.878 INFO:teuthology.orchestra.run.smithi170.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:01:54.879 INFO:teuthology.orchestra.run.smithi170.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:01:54.879 INFO:teuthology.orchestra.run.smithi170.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:01:54.879 INFO:teuthology.orchestra.run.smithi170.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:01:54.879 INFO:teuthology.orchestra.run.smithi170.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:01:54.879 INFO:teuthology.orchestra.run.smithi170.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:01:54.879 INFO:teuthology.orchestra.run.smithi170.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:01:54.879 INFO:teuthology.orchestra.run.smithi170.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:01:54.879 INFO:teuthology.orchestra.run.smithi170.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:01:54.880 INFO:teuthology.orchestra.run.smithi170.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:01:54.903 INFO:teuthology.orchestra.run.smithi170.stdout:Discarding blocks...Done. 2023-09-22T13:01:54.905 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi170.front.sepia.ceph.com -o noatime 2023-09-22T13:01:54.905 DEBUG:teuthology.orchestra.run.smithi170:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-1 2023-09-22T13:01:54.985 DEBUG:teuthology.orchestra.run.smithi170:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-1 2023-09-22T13:01:55.056 DEBUG:teuthology.orchestra.run.smithi170:> sudo mkdir -p /var/lib/ceph/osd/ceph-2 2023-09-22T13:01:55.124 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:01:55.124 INFO:tasks.ceph:role: osd.2 2023-09-22T13:01:55.124 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi170.front.sepia.ceph.com 2023-09-22T13:01:55.124 DEBUG:teuthology.orchestra.run.smithi170:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2023-09-22T13:01:55.191 INFO:teuthology.orchestra.run.smithi170.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:01:55.191 INFO:teuthology.orchestra.run.smithi170.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:01:55.191 INFO:teuthology.orchestra.run.smithi170.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:01:55.191 INFO:teuthology.orchestra.run.smithi170.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:01:55.192 INFO:teuthology.orchestra.run.smithi170.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:01:55.192 INFO:teuthology.orchestra.run.smithi170.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:01:55.192 INFO:teuthology.orchestra.run.smithi170.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:01:55.192 INFO:teuthology.orchestra.run.smithi170.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:01:55.192 INFO:teuthology.orchestra.run.smithi170.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:01:55.192 INFO:teuthology.orchestra.run.smithi170.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:01:55.214 INFO:teuthology.orchestra.run.smithi170.stdout:Discarding blocks...Done. 2023-09-22T13:01:55.215 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi170.front.sepia.ceph.com -o noatime 2023-09-22T13:01:55.216 DEBUG:teuthology.orchestra.run.smithi170:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-2 2023-09-22T13:01:55.296 DEBUG:teuthology.orchestra.run.smithi170:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-2 2023-09-22T13:01:55.366 DEBUG:teuthology.orchestra.run.smithi170:> sudo mkdir -p /var/lib/ceph/osd/ceph-3 2023-09-22T13:01:55.434 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:01:55.434 INFO:tasks.ceph:role: osd.3 2023-09-22T13:01:55.435 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi170.front.sepia.ceph.com 2023-09-22T13:01:55.435 DEBUG:teuthology.orchestra.run.smithi170:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2023-09-22T13:01:55.501 INFO:teuthology.orchestra.run.smithi170.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:01:55.501 INFO:teuthology.orchestra.run.smithi170.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:01:55.502 INFO:teuthology.orchestra.run.smithi170.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:01:55.502 INFO:teuthology.orchestra.run.smithi170.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:01:55.502 INFO:teuthology.orchestra.run.smithi170.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:01:55.502 INFO:teuthology.orchestra.run.smithi170.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:01:55.502 INFO:teuthology.orchestra.run.smithi170.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:01:55.502 INFO:teuthology.orchestra.run.smithi170.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:01:55.503 INFO:teuthology.orchestra.run.smithi170.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:01:55.503 INFO:teuthology.orchestra.run.smithi170.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:01:55.524 INFO:teuthology.orchestra.run.smithi170.stdout:Discarding blocks...Done. 2023-09-22T13:01:55.526 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi170.front.sepia.ceph.com -o noatime 2023-09-22T13:01:55.526 DEBUG:teuthology.orchestra.run.smithi170:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-3 2023-09-22T13:01:55.606 DEBUG:teuthology.orchestra.run.smithi170:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-3 2023-09-22T13:01:55.677 DEBUG:teuthology.orchestra.run.smithi170:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:01:55.772 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:55.770+0000 7f96aefa0200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2023-09-22T13:01:55.772 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:55.770+0000 7f96aefa0200 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2023-09-22T13:01:55.772 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:55.770+0000 7f96aefa0200 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2023-09-22T13:01:55.772 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:55.770+0000 7f96aefa0200 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2023-09-22T13:01:55.772 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:55.770+0000 7f96aefa0200 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2023-09-22T13:01:55.773 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:55.771+0000 7f96aefa0200 -1 bluestore(/var/lib/ceph/osd/ceph-0) _read_fsid unparsable uuid 2023-09-22T13:01:58.610 DEBUG:teuthology.orchestra.run.smithi170:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-22T13:01:58.645 DEBUG:teuthology.orchestra.run.smithi170:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:01:58.734 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:58.732+0000 7f89a3ca9200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2023-09-22T13:01:58.734 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:58.733+0000 7f89a3ca9200 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2023-09-22T13:01:58.734 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:58.733+0000 7f89a3ca9200 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2023-09-22T13:01:58.734 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:58.733+0000 7f89a3ca9200 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2023-09-22T13:01:58.735 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:58.733+0000 7f89a3ca9200 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2023-09-22T13:01:58.735 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:01:58.733+0000 7f89a3ca9200 -1 bluestore(/var/lib/ceph/osd/ceph-1) _read_fsid unparsable uuid 2023-09-22T13:02:01.577 DEBUG:teuthology.orchestra.run.smithi170:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-22T13:02:01.609 DEBUG:teuthology.orchestra.run.smithi170:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:02:01.697 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:01.696+0000 7fb6d7b15200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2023-09-22T13:02:01.698 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:01.696+0000 7fb6d7b15200 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2023-09-22T13:02:01.698 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:01.696+0000 7fb6d7b15200 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2023-09-22T13:02:01.698 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:01.696+0000 7fb6d7b15200 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2023-09-22T13:02:01.698 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:01.696+0000 7fb6d7b15200 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2023-09-22T13:02:01.699 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:01.696+0000 7fb6d7b15200 -1 bluestore(/var/lib/ceph/osd/ceph-2) _read_fsid unparsable uuid 2023-09-22T13:02:04.542 DEBUG:teuthology.orchestra.run.smithi170:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-22T13:02:04.573 DEBUG:teuthology.orchestra.run.smithi170:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:02:04.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:04.660+0000 7f2732428200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2023-09-22T13:02:04.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:04.660+0000 7f2732428200 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2023-09-22T13:02:04.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:04.660+0000 7f2732428200 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2023-09-22T13:02:04.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:04.661+0000 7f2732428200 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2023-09-22T13:02:04.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:04.661+0000 7f2732428200 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2023-09-22T13:02:04.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:02:04.661+0000 7f2732428200 -1 bluestore(/var/lib/ceph/osd/ceph-3) _read_fsid unparsable uuid 2023-09-22T13:02:07.500 DEBUG:teuthology.orchestra.run.smithi170:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-22T13:02:07.534 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/osd/ceph-4 2023-09-22T13:02:07.568 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:02:07.568 INFO:tasks.ceph:role: osd.4 2023-09-22T13:02:07.568 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi191.front.sepia.ceph.com 2023-09-22T13:02:07.569 DEBUG:teuthology.orchestra.run.smithi191:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2023-09-22T13:02:07.647 INFO:teuthology.orchestra.run.smithi191.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:02:07.647 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:02:07.647 INFO:teuthology.orchestra.run.smithi191.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:02:07.647 INFO:teuthology.orchestra.run.smithi191.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:02:07.647 INFO:teuthology.orchestra.run.smithi191.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:02:07.648 INFO:teuthology.orchestra.run.smithi191.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:02:07.648 INFO:teuthology.orchestra.run.smithi191.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:02:07.648 INFO:teuthology.orchestra.run.smithi191.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:02:07.648 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:02:07.648 INFO:teuthology.orchestra.run.smithi191.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:02:08.075 INFO:teuthology.orchestra.run.smithi191.stdout:Discarding blocks...Done. 2023-09-22T13:02:08.077 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi191.front.sepia.ceph.com -o noatime 2023-09-22T13:02:08.077 DEBUG:teuthology.orchestra.run.smithi191:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-4 2023-09-22T13:02:08.240 DEBUG:teuthology.orchestra.run.smithi191:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-4 2023-09-22T13:02:08.277 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/osd/ceph-5 2023-09-22T13:02:08.347 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:02:08.347 INFO:tasks.ceph:role: osd.5 2023-09-22T13:02:08.347 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi191.front.sepia.ceph.com 2023-09-22T13:02:08.347 DEBUG:teuthology.orchestra.run.smithi191:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2023-09-22T13:02:08.423 INFO:teuthology.orchestra.run.smithi191.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:02:08.423 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:02:08.423 INFO:teuthology.orchestra.run.smithi191.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:02:08.424 INFO:teuthology.orchestra.run.smithi191.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:02:08.424 INFO:teuthology.orchestra.run.smithi191.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:02:08.424 INFO:teuthology.orchestra.run.smithi191.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:02:08.424 INFO:teuthology.orchestra.run.smithi191.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:02:08.424 INFO:teuthology.orchestra.run.smithi191.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:02:08.424 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:02:08.425 INFO:teuthology.orchestra.run.smithi191.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:02:08.853 INFO:teuthology.orchestra.run.smithi191.stdout:Discarding blocks...Done. 2023-09-22T13:02:08.855 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi191.front.sepia.ceph.com -o noatime 2023-09-22T13:02:08.855 DEBUG:teuthology.orchestra.run.smithi191:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-5 2023-09-22T13:02:08.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-5 2023-09-22T13:02:09.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/osd/ceph-6 2023-09-22T13:02:09.074 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:02:09.074 INFO:tasks.ceph:role: osd.6 2023-09-22T13:02:09.074 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi191.front.sepia.ceph.com 2023-09-22T13:02:09.074 DEBUG:teuthology.orchestra.run.smithi191:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2023-09-22T13:02:09.150 INFO:teuthology.orchestra.run.smithi191.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:02:09.150 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:02:09.151 INFO:teuthology.orchestra.run.smithi191.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:02:09.151 INFO:teuthology.orchestra.run.smithi191.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:02:09.151 INFO:teuthology.orchestra.run.smithi191.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:02:09.151 INFO:teuthology.orchestra.run.smithi191.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:02:09.151 INFO:teuthology.orchestra.run.smithi191.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:02:09.151 INFO:teuthology.orchestra.run.smithi191.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:02:09.151 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:02:09.151 INFO:teuthology.orchestra.run.smithi191.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:02:09.592 INFO:teuthology.orchestra.run.smithi191.stdout:Discarding blocks...Done. 2023-09-22T13:02:09.594 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi191.front.sepia.ceph.com -o noatime 2023-09-22T13:02:09.594 DEBUG:teuthology.orchestra.run.smithi191:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-6 2023-09-22T13:02:09.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-6 2023-09-22T13:02:09.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/osd/ceph-7 2023-09-22T13:02:09.874 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:02:09.874 INFO:tasks.ceph:role: osd.7 2023-09-22T13:02:09.874 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi191.front.sepia.ceph.com 2023-09-22T13:02:09.874 DEBUG:teuthology.orchestra.run.smithi191:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2023-09-22T13:02:09.950 INFO:teuthology.orchestra.run.smithi191.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:02:09.950 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:02:09.951 INFO:teuthology.orchestra.run.smithi191.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:02:09.951 INFO:teuthology.orchestra.run.smithi191.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:02:09.951 INFO:teuthology.orchestra.run.smithi191.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:02:09.951 INFO:teuthology.orchestra.run.smithi191.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:02:09.951 INFO:teuthology.orchestra.run.smithi191.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:02:09.951 INFO:teuthology.orchestra.run.smithi191.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:02:09.952 INFO:teuthology.orchestra.run.smithi191.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:02:09.952 INFO:teuthology.orchestra.run.smithi191.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:02:10.382 INFO:teuthology.orchestra.run.smithi191.stdout:Discarding blocks...Done. 2023-09-22T13:02:10.384 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi191.front.sepia.ceph.com -o noatime 2023-09-22T13:02:10.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-7 2023-09-22T13:02:10.542 DEBUG:teuthology.orchestra.run.smithi191:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-7 2023-09-22T13:02:10.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 4 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:02:10.681 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:10.680+0000 7f09cc4d0200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-4/keyring: can't open /var/lib/ceph/osd/ceph-4/keyring: (2) No such file or directory 2023-09-22T13:02:10.682 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:10.680+0000 7f09cc4d0200 -1 created new key in keyring /var/lib/ceph/osd/ceph-4/keyring 2023-09-22T13:02:10.682 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:10.680+0000 7f09cc4d0200 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2023-09-22T13:02:10.682 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:10.680+0000 7f09cc4d0200 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2023-09-22T13:02:10.682 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:10.680+0000 7f09cc4d0200 -1 bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (2) No such file or directory 2023-09-22T13:02:10.682 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:10.680+0000 7f09cc4d0200 -1 bluestore(/var/lib/ceph/osd/ceph-4) _read_fsid unparsable uuid 2023-09-22T13:02:13.519 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-09-22T13:02:13.551 DEBUG:teuthology.orchestra.run.smithi191:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 5 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:02:13.639 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:13.637+0000 7f035fced200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-5/keyring: can't open /var/lib/ceph/osd/ceph-5/keyring: (2) No such file or directory 2023-09-22T13:02:13.639 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:13.638+0000 7f035fced200 -1 created new key in keyring /var/lib/ceph/osd/ceph-5/keyring 2023-09-22T13:02:13.639 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:13.638+0000 7f035fced200 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2023-09-22T13:02:13.639 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:13.638+0000 7f035fced200 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2023-09-22T13:02:13.639 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:13.638+0000 7f035fced200 -1 bluestore(/var/lib/ceph/osd/ceph-5/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-5/block: (2) No such file or directory 2023-09-22T13:02:13.640 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:13.638+0000 7f035fced200 -1 bluestore(/var/lib/ceph/osd/ceph-5) _read_fsid unparsable uuid 2023-09-22T13:02:16.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-09-22T13:02:16.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 6 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:02:16.604 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:16.602+0000 7fd4eee32200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-6/keyring: can't open /var/lib/ceph/osd/ceph-6/keyring: (2) No such file or directory 2023-09-22T13:02:16.605 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:16.602+0000 7fd4eee32200 -1 created new key in keyring /var/lib/ceph/osd/ceph-6/keyring 2023-09-22T13:02:16.605 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:16.602+0000 7fd4eee32200 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2023-09-22T13:02:16.605 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:16.602+0000 7fd4eee32200 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2023-09-22T13:02:16.605 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:16.602+0000 7fd4eee32200 -1 bluestore(/var/lib/ceph/osd/ceph-6/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-6/block: (2) No such file or directory 2023-09-22T13:02:16.605 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:16.603+0000 7fd4eee32200 -1 bluestore(/var/lib/ceph/osd/ceph-6) _read_fsid unparsable uuid 2023-09-22T13:02:19.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-22T13:02:19.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 7 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:02:19.572 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:19.571+0000 7f35103c1200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-7/keyring: can't open /var/lib/ceph/osd/ceph-7/keyring: (2) No such file or directory 2023-09-22T13:02:19.572 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:19.571+0000 7f35103c1200 -1 created new key in keyring /var/lib/ceph/osd/ceph-7/keyring 2023-09-22T13:02:19.572 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:19.571+0000 7f35103c1200 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2023-09-22T13:02:19.572 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:19.571+0000 7f35103c1200 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2023-09-22T13:02:19.573 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:19.571+0000 7f35103c1200 -1 bluestore(/var/lib/ceph/osd/ceph-7/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-7/block: (2) No such file or directory 2023-09-22T13:02:19.573 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:02:19.572+0000 7f35103c1200 -1 bluestore(/var/lib/ceph/osd/ceph-7) _read_fsid unparsable uuid 2023-09-22T13:02:22.407 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-09-22T13:02:22.438 INFO:tasks.ceph:Reading keys from all nodes... 2023-09-22T13:02:22.438 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:22.438 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/var/lib/ceph/mgr/ceph-y/keyring of=/dev/stdout 2023-09-22T13:02:22.471 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:22.472 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/var/lib/ceph/osd/ceph-0/keyring of=/dev/stdout 2023-09-22T13:02:22.543 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:22.543 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/var/lib/ceph/osd/ceph-1/keyring of=/dev/stdout 2023-09-22T13:02:22.613 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:22.613 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/var/lib/ceph/osd/ceph-2/keyring of=/dev/stdout 2023-09-22T13:02:22.682 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:22.683 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/var/lib/ceph/osd/ceph-3/keyring of=/dev/stdout 2023-09-22T13:02:22.751 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:02:22.751 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/mgr/ceph-x/keyring of=/dev/stdout 2023-09-22T13:02:22.784 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:02:22.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-4/keyring of=/dev/stdout 2023-09-22T13:02:22.855 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:02:22.855 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-5/keyring of=/dev/stdout 2023-09-22T13:02:22.924 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:02:22.924 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-6/keyring of=/dev/stdout 2023-09-22T13:02:22.994 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:02:22.994 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-7/keyring of=/dev/stdout 2023-09-22T13:02:23.063 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:23.063 DEBUG:teuthology.orchestra.run.smithi170:> dd if=/etc/ceph/ceph.client.0.keyring of=/dev/stdout 2023-09-22T13:02:23.083 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:02:23.083 DEBUG:teuthology.orchestra.run.smithi191:> dd if=/etc/ceph/ceph.client.1.keyring of=/dev/stdout 2023-09-22T13:02:23.122 INFO:tasks.ceph:Adding keys to all mons... 2023-09-22T13:02:23.122 DEBUG:teuthology.orchestra.run.smithi170:> sudo tee -a /etc/ceph/ceph.keyring 2023-09-22T13:02:23.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo tee -a /etc/ceph/ceph.keyring 2023-09-22T13:02:23.165 INFO:teuthology.orchestra.run.smithi170.stdout:[mgr.y] 2023-09-22T13:02:23.165 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBCkA1lMci3DhAAcO8cmVpwQRBz9M85hH2v5Q== 2023-09-22T13:02:23.165 INFO:teuthology.orchestra.run.smithi170.stdout:[osd.0] 2023-09-22T13:02:23.165 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBDkA1lgYYDLhAAA50eni1dH5r+EUaXEANESg== 2023-09-22T13:02:23.165 INFO:teuthology.orchestra.run.smithi170.stdout:[osd.1] 2023-09-22T13:02:23.165 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBGkA1lWD3DKxAAfnFe6UU5lAFnKwHABgF8eA== 2023-09-22T13:02:23.165 INFO:teuthology.orchestra.run.smithi170.stdout:[osd.2] 2023-09-22T13:02:23.166 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBJkA1lTWWVKRAANCo/oC2K64yfTk6EbnFV6w== 2023-09-22T13:02:23.166 INFO:teuthology.orchestra.run.smithi170.stdout:[osd.3] 2023-09-22T13:02:23.166 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBMkA1lC4FnJxAAbCAtpAp7L8zRCUY8pIbstQ== 2023-09-22T13:02:23.166 INFO:teuthology.orchestra.run.smithi170.stdout:[mgr.x] 2023-09-22T13:02:23.166 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBCkA1loN6VFRAAQv24I8RIG/zunDtcPmC1/w== 2023-09-22T13:02:23.166 INFO:teuthology.orchestra.run.smithi170.stdout:[osd.4] 2023-09-22T13:02:23.166 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBSkA1l+JOjKBAA/icn1BE3RcyHfjn0yeaVtQ== 2023-09-22T13:02:23.166 INFO:teuthology.orchestra.run.smithi170.stdout:[osd.5] 2023-09-22T13:02:23.166 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBVkA1ldTwgJhAAoGD0fbVuy+KZnFbOJVRr2Q== 2023-09-22T13:02:23.166 INFO:teuthology.orchestra.run.smithi170.stdout:[osd.6] 2023-09-22T13:02:23.167 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBYkA1l3DwIJBAAz+ojdTJ/Dn+5RXIy9HcXzg== 2023-09-22T13:02:23.167 INFO:teuthology.orchestra.run.smithi170.stdout:[osd.7] 2023-09-22T13:02:23.167 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBbkA1lN2MeIhAAMNQ2seWzL5CpRK/6OPcH/A== 2023-09-22T13:02:23.167 INFO:teuthology.orchestra.run.smithi170.stdout:[client.0] 2023-09-22T13:02:23.167 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBCkA1lgvIpGBAAGZIit1bJJ38mWCA74hp98g== 2023-09-22T13:02:23.167 INFO:teuthology.orchestra.run.smithi170.stdout:[client.1] 2023-09-22T13:02:23.167 INFO:teuthology.orchestra.run.smithi170.stdout: key = AQBCkA1lMjFIGxAAuJMDKozorxpOBRnLQ1/wOQ== 2023-09-22T13:02:23.189 INFO:teuthology.orchestra.run.smithi191.stdout:[mgr.y] 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBCkA1lMci3DhAAcO8cmVpwQRBz9M85hH2v5Q== 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.0] 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBDkA1lgYYDLhAAA50eni1dH5r+EUaXEANESg== 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.1] 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBGkA1lWD3DKxAAfnFe6UU5lAFnKwHABgF8eA== 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.2] 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBJkA1lTWWVKRAANCo/oC2K64yfTk6EbnFV6w== 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.3] 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBMkA1lC4FnJxAAbCAtpAp7L8zRCUY8pIbstQ== 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout:[mgr.x] 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBCkA1loN6VFRAAQv24I8RIG/zunDtcPmC1/w== 2023-09-22T13:02:23.190 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.4] 2023-09-22T13:02:23.191 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBSkA1l+JOjKBAA/icn1BE3RcyHfjn0yeaVtQ== 2023-09-22T13:02:23.191 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.5] 2023-09-22T13:02:23.191 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBVkA1ldTwgJhAAoGD0fbVuy+KZnFbOJVRr2Q== 2023-09-22T13:02:23.191 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.6] 2023-09-22T13:02:23.191 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBYkA1l3DwIJBAAz+ojdTJ/Dn+5RXIy9HcXzg== 2023-09-22T13:02:23.191 INFO:teuthology.orchestra.run.smithi191.stdout:[osd.7] 2023-09-22T13:02:23.191 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBbkA1lN2MeIhAAMNQ2seWzL5CpRK/6OPcH/A== 2023-09-22T13:02:23.191 INFO:teuthology.orchestra.run.smithi191.stdout:[client.0] 2023-09-22T13:02:23.191 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBCkA1lgvIpGBAAGZIit1bJJ38mWCA74hp98g== 2023-09-22T13:02:23.191 INFO:teuthology.orchestra.run.smithi191.stdout:[client.1] 2023-09-22T13:02:23.191 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBCkA1lMjFIGxAAuJMDKozorxpOBRnLQ1/wOQ== 2023-09-22T13:02:23.192 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:02:23.210 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:02:23.282 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.333 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.335 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.384 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.435 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.437 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.485 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:02:23.487 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:02:23.535 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.537 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.586 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.588 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.636 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.638 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.687 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:02:23.738 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:02:23.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:02:23.792 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:02:23.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:02:23.845 INFO:tasks.ceph:Running mkfs on mon nodes... 2023-09-22T13:02:23.846 DEBUG:teuthology.orchestra.run.smithi170:> sudo mkdir -p /var/lib/ceph/mon/ceph-a 2023-09-22T13:02:23.872 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i a --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2023-09-22T13:02:23.965 DEBUG:teuthology.orchestra.run.smithi170:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-a 2023-09-22T13:02:23.992 DEBUG:teuthology.orchestra.run.smithi170:> sudo mkdir -p /var/lib/ceph/mon/ceph-c 2023-09-22T13:02:24.060 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i c --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2023-09-22T13:02:24.153 DEBUG:teuthology.orchestra.run.smithi170:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-c 2023-09-22T13:02:24.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /var/lib/ceph/mon/ceph-b 2023-09-22T13:02:24.208 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i b --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2023-09-22T13:02:24.302 DEBUG:teuthology.orchestra.run.smithi191:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-b 2023-09-22T13:02:24.329 DEBUG:teuthology.orchestra.run.smithi170:> rm -- /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:02:24.331 DEBUG:teuthology.orchestra.run.smithi191:> rm -- /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:02:24.388 INFO:tasks.ceph:Starting mon daemons in cluster ceph... 2023-09-22T13:02:24.388 INFO:tasks.ceph.mon.a:Restarting daemon 2023-09-22T13:02:24.389 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2023-09-22T13:02:24.391 INFO:tasks.ceph.mon.a:Started 2023-09-22T13:02:24.391 INFO:tasks.ceph.mon.c:Restarting daemon 2023-09-22T13:02:24.391 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2023-09-22T13:02:24.393 INFO:tasks.ceph.mon.c:Started 2023-09-22T13:02:24.394 INFO:tasks.ceph.mon.b:Restarting daemon 2023-09-22T13:02:24.394 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2023-09-22T13:02:24.430 INFO:tasks.ceph.mon.b:Started 2023-09-22T13:02:24.431 INFO:tasks.ceph:Starting mgr daemons in cluster ceph... 2023-09-22T13:02:24.431 INFO:tasks.ceph.mgr.y:Restarting daemon 2023-09-22T13:02:24.431 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2023-09-22T13:02:24.434 INFO:tasks.ceph.mgr.y:Started 2023-09-22T13:02:24.434 INFO:tasks.ceph.mgr.x:Restarting daemon 2023-09-22T13:02:24.434 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2023-09-22T13:02:24.436 INFO:tasks.ceph.mgr.x:Started 2023-09-22T13:02:24.437 INFO:tasks.ceph:Setting crush tunables to default 2023-09-22T13:02:24.437 DEBUG:teuthology.orchestra.run.smithi170:> sudo ceph --cluster ceph osd crush tunables default 2023-09-22T13:02:29.821 INFO:teuthology.orchestra.run.smithi170.stderr:adjusted tunables profile to default 2023-09-22T13:02:29.834 INFO:tasks.ceph:Starting osd daemons in cluster ceph... 2023-09-22T13:02:29.834 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:29.835 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/var/lib/ceph/osd/ceph-0/fsid of=/dev/stdout 2023-09-22T13:02:29.862 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:29.862 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/var/lib/ceph/osd/ceph-1/fsid of=/dev/stdout 2023-09-22T13:02:29.871 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:29.871+0000 7f5243c1e680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-22T13:02:29.873 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:29.872+0000 7f28dbaf2680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-22T13:02:29.897 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:29.897 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/var/lib/ceph/osd/ceph-2/fsid of=/dev/stdout 2023-09-22T13:02:29.941 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:29.941+0000 7f28dbaf2680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-22T13:02:29.944 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:29.943+0000 7f5243c1e680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-22T13:02:29.965 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:29.966 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd if=/var/lib/ceph/osd/ceph-3/fsid of=/dev/stdout 2023-09-22T13:02:30.032 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:02:30.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-4/fsid of=/dev/stdout 2023-09-22T13:02:30.059 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:02:30.059 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-5/fsid of=/dev/stdout 2023-09-22T13:02:30.127 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:02:30.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-6/fsid of=/dev/stdout 2023-09-22T13:02:30.195 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2023-09-22T13:02:30.195 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/var/lib/ceph/osd/ceph-7/fsid of=/dev/stdout 2023-09-22T13:02:30.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new fe22ddd2-5eb8-47b5-ba38-0c16e178a5d4 0 2023-09-22T13:02:30.348 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:30.347+0000 7f5243c1e680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-22T13:02:30.390 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:30.390+0000 7f28dbaf2680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-22T13:02:30.513 INFO:teuthology.orchestra.run.smithi191.stdout:0 2023-09-22T13:02:30.526 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new decb9642-a16b-4bce-a715-e8bb86d57316 1 2023-09-22T13:02:30.768 INFO:teuthology.orchestra.run.smithi191.stdout:1 2023-09-22T13:02:30.781 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new b4ebd650-1e03-498d-8c0e-b241c43e5c3a 2 2023-09-22T13:02:30.999 INFO:teuthology.orchestra.run.smithi191.stdout:2 2023-09-22T13:02:31.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new f97e19c6-a762-486b-8b70-47eeb70dfcc3 3 2023-09-22T13:02:31.081 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:31.080+0000 7f5243c1e680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-22T13:02:31.147 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:31.146+0000 7f5243c1e680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-22T13:02:31.183 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:31.183+0000 7f28dbaf2680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-22T13:02:31.246 INFO:teuthology.orchestra.run.smithi191.stdout:3 2023-09-22T13:02:31.252 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:31.252+0000 7f28dbaf2680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-22T13:02:31.259 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new 3a10cacd-01da-4e94-a001-361dfce676bd 4 2023-09-22T13:02:31.336 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:31.335+0000 7f5243c1e680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-22T13:02:31.452 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:31.452+0000 7f28dbaf2680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-22T13:02:31.473 INFO:teuthology.orchestra.run.smithi191.stdout:4 2023-09-22T13:02:31.485 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new a8bd46a3-dd7f-4df2-a93b-61a6ac38621e 5 2023-09-22T13:02:31.745 INFO:teuthology.orchestra.run.smithi191.stdout:5 2023-09-22T13:02:31.757 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new 320479ef-0a92-4b8b-bd4c-2162f512c8c8 6 2023-09-22T13:02:31.774 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:31.774+0000 7f5243c1e680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-22T13:02:31.875 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:31.875+0000 7f5243c1e680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:02:31.947 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:31.947+0000 7f28dbaf2680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-22T13:02:31.967 INFO:teuthology.orchestra.run.smithi191.stdout:6 2023-09-22T13:02:31.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph --cluster ceph osd new 7b27eaf0-ac40-41b0-af69-1fc378ddbe64 7 2023-09-22T13:02:32.032 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:32.031+0000 7f5243c1e680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-22T13:02:32.055 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:32.055+0000 7f28dbaf2680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:02:32.108 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:32.108+0000 7f5243c1e680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-22T13:02:32.178 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:32.177+0000 7f5243c1e680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-22T13:02:32.189 INFO:teuthology.orchestra.run.smithi191.stdout:7 2023-09-22T13:02:32.201 INFO:tasks.ceph.osd.0:Restarting daemon 2023-09-22T13:02:32.201 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2023-09-22T13:02:32.203 INFO:tasks.ceph.osd.0:Started 2023-09-22T13:02:32.203 INFO:tasks.ceph.osd.1:Restarting daemon 2023-09-22T13:02:32.203 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2023-09-22T13:02:32.205 INFO:tasks.ceph.osd.1:Started 2023-09-22T13:02:32.205 INFO:tasks.ceph.osd.2:Restarting daemon 2023-09-22T13:02:32.206 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2023-09-22T13:02:32.208 INFO:tasks.ceph.osd.2:Started 2023-09-22T13:02:32.208 INFO:tasks.ceph.osd.3:Restarting daemon 2023-09-22T13:02:32.208 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2023-09-22T13:02:32.210 INFO:tasks.ceph.osd.3:Started 2023-09-22T13:02:32.210 INFO:tasks.ceph.osd.4:Restarting daemon 2023-09-22T13:02:32.210 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 4 2023-09-22T13:02:32.212 INFO:tasks.ceph.osd.4:Started 2023-09-22T13:02:32.213 INFO:tasks.ceph.osd.5:Restarting daemon 2023-09-22T13:02:32.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 5 2023-09-22T13:02:32.215 INFO:tasks.ceph.osd.5:Started 2023-09-22T13:02:32.215 INFO:tasks.ceph.osd.6:Restarting daemon 2023-09-22T13:02:32.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 6 2023-09-22T13:02:32.217 INFO:tasks.ceph.osd.6:Started 2023-09-22T13:02:32.217 INFO:tasks.ceph.osd.7:Restarting daemon 2023-09-22T13:02:32.217 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 7 2023-09-22T13:02:32.218 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:32.217+0000 7f28dbaf2680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-22T13:02:32.220 INFO:tasks.ceph.osd.7:Started 2023-09-22T13:02:32.220 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-09-22T13:02:32.329 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:32.328+0000 7f28dbaf2680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-22T13:02:32.398 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:32.398+0000 7f28dbaf2680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-22T13:02:32.460 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:02:32.460 INFO:teuthology.orchestra.run.smithi170.stdout:{"epoch":10,"fsid":"3f2e844c-1843-4fab-beef-461ea345cf51","created":"2023-09-22T13:02:29.721682+0000","modified":"2023-09-22T13:02:32.184793+0000","last_up_change":"0.000000","last_in_change":"2023-09-22T13:02:32.184793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[],"osds":[{"osd":0,"uuid":"fe22ddd2-5eb8-47b5-ba38-0c16e178a5d4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"decb9642-a16b-4bce-a715-e8bb86d57316","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"b4ebd650-1e03-498d-8c0e-b241c43e5c3a","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"f97e19c6-a762-486b-8b70-47eeb70dfcc3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"3a10cacd-01da-4e94-a001-361dfce676bd","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"a8bd46a3-dd7f-4df2-a93b-61a6ac38621e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"320479ef-0a92-4b8b-bd4c-2162f512c8c8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"7b27eaf0-ac40-41b0-af69-1fc378ddbe64","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:02:32.472 INFO:tasks.ceph.ceph_manager.ceph:[] 2023-09-22T13:02:32.472 INFO:tasks.ceph:Waiting for OSDs to come up 2023-09-22T13:02:32.483 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:32.482+0000 7f5243c1e680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-22T13:02:32.657 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:32.657+0000 7f28dbaf2680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-22T13:02:32.681 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:32.680+0000 7f5243c1e680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-22T13:02:32.816 INFO:tasks.ceph.osd.4.smithi191.stderr:2023-09-22T13:02:32.815+0000 7f521bd78200 -1 Falling back to public interface 2023-09-22T13:02:32.816 INFO:tasks.ceph.osd.5.smithi191.stderr:2023-09-22T13:02:32.816+0000 7f14cd017200 -1 Falling back to public interface 2023-09-22T13:02:32.820 INFO:tasks.ceph.osd.7.smithi191.stderr:2023-09-22T13:02:32.819+0000 7f0d1c9e2200 -1 Falling back to public interface 2023-09-22T13:02:32.826 INFO:tasks.ceph.osd.1.smithi170.stderr:2023-09-22T13:02:32.826+0000 7f85e9eef200 -1 Falling back to public interface 2023-09-22T13:02:32.830 INFO:tasks.ceph.osd.0.smithi170.stderr:2023-09-22T13:02:32.829+0000 7fa192de4200 -1 Falling back to public interface 2023-09-22T13:02:32.831 INFO:tasks.ceph.osd.2.smithi170.stderr:2023-09-22T13:02:32.830+0000 7f27093f3200 -1 Falling back to public interface 2023-09-22T13:02:32.832 INFO:tasks.ceph.osd.6.smithi191.stderr:2023-09-22T13:02:32.832+0000 7eff8b9f8200 -1 Falling back to public interface 2023-09-22T13:02:32.839 INFO:tasks.ceph.osd.3.smithi170.stderr:2023-09-22T13:02:32.838+0000 7f2112744200 -1 Falling back to public interface 2023-09-22T13:02:32.856 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:32.856+0000 7f28dbaf2680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-22T13:02:33.278 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-22T13:02:33.356 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:33.355+0000 7f5243c1e680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-22T13:02:33.429 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:33.428+0000 7f5243c1e680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-22T13:02:33.464 INFO:teuthology.misc.health.smithi170.stdout: 2023-09-22T13:02:33.464 INFO:teuthology.misc.health.smithi170.stdout:{"epoch":10,"fsid":"3f2e844c-1843-4fab-beef-461ea345cf51","created":"2023-09-22T13:02:29.721682+0000","modified":"2023-09-22T13:02:32.184793+0000","last_up_change":"0.000000","last_in_change":"2023-09-22T13:02:32.184793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[],"osds":[{"osd":0,"uuid":"fe22ddd2-5eb8-47b5-ba38-0c16e178a5d4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"decb9642-a16b-4bce-a715-e8bb86d57316","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"b4ebd650-1e03-498d-8c0e-b241c43e5c3a","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"f97e19c6-a762-486b-8b70-47eeb70dfcc3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"3a10cacd-01da-4e94-a001-361dfce676bd","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"a8bd46a3-dd7f-4df2-a93b-61a6ac38621e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"320479ef-0a92-4b8b-bd4c-2162f512c8c8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"7b27eaf0-ac40-41b0-af69-1fc378ddbe64","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:02:33.475 DEBUG:teuthology.misc:0 of 8 OSDs are up 2023-09-22T13:02:33.509 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:33.508+0000 7f5243c1e680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-22T13:02:33.521 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:33.521+0000 7f28dbaf2680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-22T13:02:33.590 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:33.589+0000 7f28dbaf2680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-22T13:02:33.657 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:33.656+0000 7f5243c1e680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-22T13:02:33.670 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:33.669+0000 7f28dbaf2680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-22T13:02:33.728 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:33.727+0000 7f5243c1e680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-22T13:02:33.816 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:33.816+0000 7f28dbaf2680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-22T13:02:33.845 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:33.844+0000 7f5243c1e680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-22T13:02:33.887 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:33.886+0000 7f28dbaf2680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-22T13:02:33.953 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:33.952+0000 7f5243c1e680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:02:34.004 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:34.004+0000 7f28dbaf2680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-22T13:02:34.111 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:34.111+0000 7f28dbaf2680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:02:34.276 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:34.275+0000 7f5243c1e680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-22T13:02:34.346 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:34.345+0000 7f5243c1e680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-22T13:02:34.439 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:34.439+0000 7f28dbaf2680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-22T13:02:34.512 INFO:tasks.ceph.mgr.x.smithi191.stderr:2023-09-22T13:02:34.512+0000 7f28dbaf2680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-22T13:02:35.370 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:35.369+0000 7f51f9119700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-22T13:02:35.371 INFO:tasks.ceph.mgr.y.smithi170.stderr:2023-09-22T13:02:35.370+0000 7f51f9119700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-22T13:02:36.271 INFO:tasks.ceph.osd.0.smithi170.stderr:2023-09-22T13:02:36.271+0000 7fa192de4200 -1 osd.0 0 log_to_monitors {default=true} 2023-09-22T13:02:36.315 INFO:tasks.ceph.osd.4.smithi191.stderr:2023-09-22T13:02:36.315+0000 7f521bd78200 -1 osd.4 0 log_to_monitors {default=true} 2023-09-22T13:02:36.323 INFO:tasks.ceph.osd.7.smithi191.stderr:2023-09-22T13:02:36.322+0000 7f0d1c9e2200 -1 osd.7 0 log_to_monitors {default=true} 2023-09-22T13:02:36.324 INFO:tasks.ceph.osd.5.smithi191.stderr:2023-09-22T13:02:36.318+0000 7f14cd017200 -1 osd.5 0 log_to_monitors {default=true} 2023-09-22T13:02:36.325 INFO:tasks.ceph.osd.6.smithi191.stderr:2023-09-22T13:02:36.325+0000 7eff8b9f8200 -1 osd.6 0 log_to_monitors {default=true} 2023-09-22T13:02:36.339 INFO:tasks.ceph.osd.3.smithi170.stderr:2023-09-22T13:02:36.338+0000 7f2112744200 -1 osd.3 0 log_to_monitors {default=true} 2023-09-22T13:02:36.609 INFO:tasks.ceph.osd.1.smithi170.stderr:2023-09-22T13:02:36.609+0000 7f85e9eef200 -1 osd.1 0 log_to_monitors {default=true} 2023-09-22T13:02:36.612 INFO:tasks.ceph.osd.2.smithi170.stderr:2023-09-22T13:02:36.611+0000 7f27093f3200 -1 osd.2 0 log_to_monitors {default=true} 2023-09-22T13:02:37.372 INFO:tasks.ceph.osd.6.smithi191.stderr:2023-09-22T13:02:37.372+0000 7eff82399700 -1 osd.6 0 waiting for initial osdmap 2023-09-22T13:02:37.375 INFO:tasks.ceph.osd.4.smithi191.stderr:2023-09-22T13:02:37.375+0000 7f5212719700 -1 osd.4 0 waiting for initial osdmap 2023-09-22T13:02:37.376 INFO:tasks.ceph.osd.7.smithi191.stderr:2023-09-22T13:02:37.375+0000 7f0d13383700 -1 osd.7 0 waiting for initial osdmap 2023-09-22T13:02:37.376 INFO:tasks.ceph.osd.5.smithi191.stderr:2023-09-22T13:02:37.376+0000 7f14c39b8700 -1 osd.5 0 waiting for initial osdmap 2023-09-22T13:02:37.379 INFO:tasks.ceph.osd.6.smithi191.stderr:2023-09-22T13:02:37.378+0000 7eff7eb15700 -1 osd.6 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:02:37.379 INFO:tasks.ceph.osd.0.smithi170.stderr:2023-09-22T13:02:37.379+0000 7fa189785700 -1 osd.0 0 waiting for initial osdmap 2023-09-22T13:02:37.379 INFO:tasks.ceph.osd.3.smithi170.stderr:2023-09-22T13:02:37.379+0000 7f21090e5700 -1 osd.3 0 waiting for initial osdmap 2023-09-22T13:02:37.385 INFO:tasks.ceph.osd.0.smithi170.stderr:2023-09-22T13:02:37.384+0000 7fa185f01700 -1 osd.0 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:02:37.385 INFO:tasks.ceph.osd.3.smithi170.stderr:2023-09-22T13:02:37.384+0000 7f2105861700 -1 osd.3 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:02:37.386 INFO:tasks.ceph.osd.5.smithi191.stderr:2023-09-22T13:02:37.386+0000 7f14c0134700 -1 osd.5 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:02:37.386 INFO:tasks.ceph.osd.4.smithi191.stderr:2023-09-22T13:02:37.386+0000 7f520ee95700 -1 osd.4 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:02:37.387 INFO:tasks.ceph.osd.7.smithi191.stderr:2023-09-22T13:02:37.386+0000 7f0d0faff700 -1 osd.7 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:02:38.375 INFO:tasks.ceph.osd.2.smithi170.stderr:2023-09-22T13:02:38.374+0000 7f26ffd94700 -1 osd.2 0 waiting for initial osdmap 2023-09-22T13:02:38.375 INFO:tasks.ceph.osd.1.smithi170.stderr:2023-09-22T13:02:38.375+0000 7f85e0890700 -1 osd.1 0 waiting for initial osdmap 2023-09-22T13:02:38.383 INFO:tasks.ceph.osd.2.smithi170.stderr:2023-09-22T13:02:38.382+0000 7f26fc510700 -1 osd.2 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:02:38.384 INFO:tasks.ceph.osd.1.smithi170.stderr:2023-09-22T13:02:38.382+0000 7f85dd00c700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:02:40.281 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-22T13:02:40.580 INFO:teuthology.misc.health.smithi170.stdout: 2023-09-22T13:02:40.580 INFO:teuthology.misc.health.smithi170.stdout:{"epoch":15,"fsid":"3f2e844c-1843-4fab-beef-461ea345cf51","created":"2023-09-22T13:02:29.721682+0000","modified":"2023-09-22T13:02:40.370518+0000","last_up_change":"2023-09-22T13:02:39.369147+0000","last_in_change":"2023-09-22T13:02:32.184793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-22T13:02:38.376015+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"fe22ddd2-5eb8-47b5-ba38-0c16e178a5d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6808","nonce":145027},{"type":"v1","addr":"172.21.15.170:6809","nonce":145027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6810","nonce":145027},{"type":"v1","addr":"172.21.15.170:6811","nonce":145027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6815","nonce":145027},{"type":"v1","addr":"172.21.15.170:6817","nonce":145027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6812","nonce":145027},{"type":"v1","addr":"172.21.15.170:6813","nonce":145027}]},"public_addr":"172.21.15.170:6809/145027","cluster_addr":"172.21.15.170:6811/145027","heartbeat_back_addr":"172.21.15.170:6817/145027","heartbeat_front_addr":"172.21.15.170:6813/145027","state":["exists","up"]},{"osd":1,"uuid":"decb9642-a16b-4bce-a715-e8bb86d57316","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6800","nonce":145025},{"type":"v1","addr":"172.21.15.170:6801","nonce":145025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6802","nonce":145025},{"type":"v1","addr":"172.21.15.170:6803","nonce":145025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6806","nonce":145025},{"type":"v1","addr":"172.21.15.170:6807","nonce":145025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6804","nonce":145025},{"type":"v1","addr":"172.21.15.170:6805","nonce":145025}]},"public_addr":"172.21.15.170:6801/145025","cluster_addr":"172.21.15.170:6803/145025","heartbeat_back_addr":"172.21.15.170:6807/145025","heartbeat_front_addr":"172.21.15.170:6805/145025","state":["exists","up"]},{"osd":2,"uuid":"b4ebd650-1e03-498d-8c0e-b241c43e5c3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6814","nonce":145026},{"type":"v1","addr":"172.21.15.170:6816","nonce":145026}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6818","nonce":145026},{"type":"v1","addr":"172.21.15.170:6819","nonce":145026}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6822","nonce":145026},{"type":"v1","addr":"172.21.15.170:6823","nonce":145026}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6820","nonce":145026},{"type":"v1","addr":"172.21.15.170:6821","nonce":145026}]},"public_addr":"172.21.15.170:6816/145026","cluster_addr":"172.21.15.170:6819/145026","heartbeat_back_addr":"172.21.15.170:6823/145026","heartbeat_front_addr":"172.21.15.170:6821/145026","state":["exists","up"]},{"osd":3,"uuid":"f97e19c6-a762-486b-8b70-47eeb70dfcc3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6824","nonce":145036},{"type":"v1","addr":"172.21.15.170:6825","nonce":145036}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6826","nonce":145036},{"type":"v1","addr":"172.21.15.170:6827","nonce":145036}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6830","nonce":145036},{"type":"v1","addr":"172.21.15.170:6831","nonce":145036}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6828","nonce":145036},{"type":"v1","addr":"172.21.15.170:6829","nonce":145036}]},"public_addr":"172.21.15.170:6825/145036","cluster_addr":"172.21.15.170:6827/145036","heartbeat_back_addr":"172.21.15.170:6831/145036","heartbeat_front_addr":"172.21.15.170:6829/145036","state":["exists","up"]},{"osd":4,"uuid":"3a10cacd-01da-4e94-a001-361dfce676bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":144990},{"type":"v1","addr":"172.21.15.191:6801","nonce":144990}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":144990},{"type":"v1","addr":"172.21.15.191:6803","nonce":144990}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":144990},{"type":"v1","addr":"172.21.15.191:6807","nonce":144990}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":144990},{"type":"v1","addr":"172.21.15.191:6805","nonce":144990}]},"public_addr":"172.21.15.191:6801/144990","cluster_addr":"172.21.15.191:6803/144990","heartbeat_back_addr":"172.21.15.191:6807/144990","heartbeat_front_addr":"172.21.15.191:6805/144990","state":["exists","up"]},{"osd":5,"uuid":"a8bd46a3-dd7f-4df2-a93b-61a6ac38621e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":144991},{"type":"v1","addr":"172.21.15.191:6809","nonce":144991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":144991},{"type":"v1","addr":"172.21.15.191:6811","nonce":144991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":144991},{"type":"v1","addr":"172.21.15.191:6815","nonce":144991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":144991},{"type":"v1","addr":"172.21.15.191:6813","nonce":144991}]},"public_addr":"172.21.15.191:6809/144991","cluster_addr":"172.21.15.191:6811/144991","heartbeat_back_addr":"172.21.15.191:6815/144991","heartbeat_front_addr":"172.21.15.191:6813/144991","state":["exists","up"]},{"osd":6,"uuid":"320479ef-0a92-4b8b-bd4c-2162f512c8c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":145036},{"type":"v1","addr":"172.21.15.191:6825","nonce":145036}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":145036},{"type":"v1","addr":"172.21.15.191:6827","nonce":145036}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":145036},{"type":"v1","addr":"172.21.15.191:6831","nonce":145036}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":145036},{"type":"v1","addr":"172.21.15.191:6829","nonce":145036}]},"public_addr":"172.21.15.191:6825/145036","cluster_addr":"172.21.15.191:6827/145036","heartbeat_back_addr":"172.21.15.191:6831/145036","heartbeat_front_addr":"172.21.15.191:6829/145036","state":["exists","up"]},{"osd":7,"uuid":"7b27eaf0-ac40-41b0-af69-1fc378ddbe64","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":144992},{"type":"v1","addr":"172.21.15.191:6817","nonce":144992}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":144992},{"type":"v1","addr":"172.21.15.191:6819","nonce":144992}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":144992},{"type":"v1","addr":"172.21.15.191:6823","nonce":144992}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":144992},{"type":"v1","addr":"172.21.15.191:6821","nonce":144992}]},"public_addr":"172.21.15.191:6817/144992","cluster_addr":"172.21.15.191:6819/144992","heartbeat_back_addr":"172.21.15.191:6823/144992","heartbeat_front_addr":"172.21.15.191:6821/144992","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.317401+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.607854+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.593088+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.307454+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.354287+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.367514+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.280836+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.284268+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:02:40.592 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-22T13:02:40.593 INFO:tasks.ceph:Creating RBD pool 2023-09-22T13:02:40.593 DEBUG:teuthology.orchestra.run.smithi170:> sudo ceph --cluster ceph osd pool create rbd 8 2023-09-22T13:02:41.495 INFO:teuthology.orchestra.run.smithi170.stderr:pool 'rbd' created 2023-09-22T13:02:41.508 DEBUG:teuthology.orchestra.run.smithi170:> sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-22T13:02:42.500 INFO:teuthology.orchestra.run.smithi170.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-22T13:02:42.515 INFO:tasks.ceph:Starting mds daemons in cluster ceph... 2023-09-22T13:02:42.515 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-22T13:02:42.515 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2023-09-22T13:02:42.516 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2023-09-22T13:02:42.516 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2023-09-22T13:02:42.853 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:02:42.865 INFO:teuthology.orchestra.run.smithi170.stdout:{"epoch":4,"active_gid":4107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6832","nonce":144727},{"type":"v1","addr":"172.21.15.170:6833","nonce":144727}]},"active_addr":"172.21.15.170:6833/144727","active_change":"2023-09-22T13:02:34.360049+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":4102,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.170:0","nonce":3142583566}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.170:0","nonce":163298170}]}]} 2023-09-22T13:02:42.870 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2023-09-22T13:02:42.870 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2023-09-22T13:02:42.870 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-09-22T13:02:43.159 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:02:43.159 INFO:teuthology.orchestra.run.smithi170.stdout:{"epoch":17,"fsid":"3f2e844c-1843-4fab-beef-461ea345cf51","created":"2023-09-22T13:02:29.721682+0000","modified":"2023-09-22T13:02:42.490415+0000","last_up_change":"2023-09-22T13:02:39.369147+0000","last_in_change":"2023-09-22T13:02:32.184793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-22T13:02:38.376015+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-22T13:02:40.881009+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"fe22ddd2-5eb8-47b5-ba38-0c16e178a5d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6808","nonce":145027},{"type":"v1","addr":"172.21.15.170:6809","nonce":145027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6810","nonce":145027},{"type":"v1","addr":"172.21.15.170:6811","nonce":145027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6815","nonce":145027},{"type":"v1","addr":"172.21.15.170:6817","nonce":145027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6812","nonce":145027},{"type":"v1","addr":"172.21.15.170:6813","nonce":145027}]},"public_addr":"172.21.15.170:6809/145027","cluster_addr":"172.21.15.170:6811/145027","heartbeat_back_addr":"172.21.15.170:6817/145027","heartbeat_front_addr":"172.21.15.170:6813/145027","state":["exists","up"]},{"osd":1,"uuid":"decb9642-a16b-4bce-a715-e8bb86d57316","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6800","nonce":145025},{"type":"v1","addr":"172.21.15.170:6801","nonce":145025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6802","nonce":145025},{"type":"v1","addr":"172.21.15.170:6803","nonce":145025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6806","nonce":145025},{"type":"v1","addr":"172.21.15.170:6807","nonce":145025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6804","nonce":145025},{"type":"v1","addr":"172.21.15.170:6805","nonce":145025}]},"public_addr":"172.21.15.170:6801/145025","cluster_addr":"172.21.15.170:6803/145025","heartbeat_back_addr":"172.21.15.170:6807/145025","heartbeat_front_addr":"172.21.15.170:6805/145025","state":["exists","up"]},{"osd":2,"uuid":"b4ebd650-1e03-498d-8c0e-b241c43e5c3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6814","nonce":145026},{"type":"v1","addr":"172.21.15.170:6816","nonce":145026}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6818","nonce":145026},{"type":"v1","addr":"172.21.15.170:6819","nonce":145026}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6822","nonce":145026},{"type":"v1","addr":"172.21.15.170:6823","nonce":145026}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6820","nonce":145026},{"type":"v1","addr":"172.21.15.170:6821","nonce":145026}]},"public_addr":"172.21.15.170:6816/145026","cluster_addr":"172.21.15.170:6819/145026","heartbeat_back_addr":"172.21.15.170:6823/145026","heartbeat_front_addr":"172.21.15.170:6821/145026","state":["exists","up"]},{"osd":3,"uuid":"f97e19c6-a762-486b-8b70-47eeb70dfcc3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6824","nonce":145036},{"type":"v1","addr":"172.21.15.170:6825","nonce":145036}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6826","nonce":145036},{"type":"v1","addr":"172.21.15.170:6827","nonce":145036}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6830","nonce":145036},{"type":"v1","addr":"172.21.15.170:6831","nonce":145036}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6828","nonce":145036},{"type":"v1","addr":"172.21.15.170:6829","nonce":145036}]},"public_addr":"172.21.15.170:6825/145036","cluster_addr":"172.21.15.170:6827/145036","heartbeat_back_addr":"172.21.15.170:6831/145036","heartbeat_front_addr":"172.21.15.170:6829/145036","state":["exists","up"]},{"osd":4,"uuid":"3a10cacd-01da-4e94-a001-361dfce676bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":144990},{"type":"v1","addr":"172.21.15.191:6801","nonce":144990}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":144990},{"type":"v1","addr":"172.21.15.191:6803","nonce":144990}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":144990},{"type":"v1","addr":"172.21.15.191:6807","nonce":144990}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":144990},{"type":"v1","addr":"172.21.15.191:6805","nonce":144990}]},"public_addr":"172.21.15.191:6801/144990","cluster_addr":"172.21.15.191:6803/144990","heartbeat_back_addr":"172.21.15.191:6807/144990","heartbeat_front_addr":"172.21.15.191:6805/144990","state":["exists","up"]},{"osd":5,"uuid":"a8bd46a3-dd7f-4df2-a93b-61a6ac38621e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":144991},{"type":"v1","addr":"172.21.15.191:6809","nonce":144991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":144991},{"type":"v1","addr":"172.21.15.191:6811","nonce":144991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":144991},{"type":"v1","addr":"172.21.15.191:6815","nonce":144991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":144991},{"type":"v1","addr":"172.21.15.191:6813","nonce":144991}]},"public_addr":"172.21.15.191:6809/144991","cluster_addr":"172.21.15.191:6811/144991","heartbeat_back_addr":"172.21.15.191:6815/144991","heartbeat_front_addr":"172.21.15.191:6813/144991","state":["exists","up"]},{"osd":6,"uuid":"320479ef-0a92-4b8b-bd4c-2162f512c8c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":145036},{"type":"v1","addr":"172.21.15.191:6825","nonce":145036}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":145036},{"type":"v1","addr":"172.21.15.191:6827","nonce":145036}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":145036},{"type":"v1","addr":"172.21.15.191:6831","nonce":145036}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":145036},{"type":"v1","addr":"172.21.15.191:6829","nonce":145036}]},"public_addr":"172.21.15.191:6825/145036","cluster_addr":"172.21.15.191:6827/145036","heartbeat_back_addr":"172.21.15.191:6831/145036","heartbeat_front_addr":"172.21.15.191:6829/145036","state":["exists","up"]},{"osd":7,"uuid":"7b27eaf0-ac40-41b0-af69-1fc378ddbe64","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":144992},{"type":"v1","addr":"172.21.15.191:6817","nonce":144992}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":144992},{"type":"v1","addr":"172.21.15.191:6819","nonce":144992}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":144992},{"type":"v1","addr":"172.21.15.191:6823","nonce":144992}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":144992},{"type":"v1","addr":"172.21.15.191:6821","nonce":144992}]},"public_addr":"172.21.15.191:6817/144992","cluster_addr":"172.21.15.191:6819/144992","heartbeat_back_addr":"172.21.15.191:6823/144992","heartbeat_front_addr":"172.21.15.191:6821/144992","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.317401+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.607854+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.593088+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.307454+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.354287+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.367514+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.280836+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.284268+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:02:43.172 INFO:tasks.ceph.ceph_manager.ceph:all up! 2023-09-22T13:02:43.173 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-09-22T13:02:43.463 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:02:43.463 INFO:teuthology.orchestra.run.smithi170.stdout:{"epoch":17,"fsid":"3f2e844c-1843-4fab-beef-461ea345cf51","created":"2023-09-22T13:02:29.721682+0000","modified":"2023-09-22T13:02:42.490415+0000","last_up_change":"2023-09-22T13:02:39.369147+0000","last_in_change":"2023-09-22T13:02:32.184793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-22T13:02:38.376015+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-22T13:02:40.881009+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"fe22ddd2-5eb8-47b5-ba38-0c16e178a5d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6808","nonce":145027},{"type":"v1","addr":"172.21.15.170:6809","nonce":145027}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6810","nonce":145027},{"type":"v1","addr":"172.21.15.170:6811","nonce":145027}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6815","nonce":145027},{"type":"v1","addr":"172.21.15.170:6817","nonce":145027}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6812","nonce":145027},{"type":"v1","addr":"172.21.15.170:6813","nonce":145027}]},"public_addr":"172.21.15.170:6809/145027","cluster_addr":"172.21.15.170:6811/145027","heartbeat_back_addr":"172.21.15.170:6817/145027","heartbeat_front_addr":"172.21.15.170:6813/145027","state":["exists","up"]},{"osd":1,"uuid":"decb9642-a16b-4bce-a715-e8bb86d57316","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6800","nonce":145025},{"type":"v1","addr":"172.21.15.170:6801","nonce":145025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6802","nonce":145025},{"type":"v1","addr":"172.21.15.170:6803","nonce":145025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6806","nonce":145025},{"type":"v1","addr":"172.21.15.170:6807","nonce":145025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6804","nonce":145025},{"type":"v1","addr":"172.21.15.170:6805","nonce":145025}]},"public_addr":"172.21.15.170:6801/145025","cluster_addr":"172.21.15.170:6803/145025","heartbeat_back_addr":"172.21.15.170:6807/145025","heartbeat_front_addr":"172.21.15.170:6805/145025","state":["exists","up"]},{"osd":2,"uuid":"b4ebd650-1e03-498d-8c0e-b241c43e5c3a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6814","nonce":145026},{"type":"v1","addr":"172.21.15.170:6816","nonce":145026}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6818","nonce":145026},{"type":"v1","addr":"172.21.15.170:6819","nonce":145026}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6822","nonce":145026},{"type":"v1","addr":"172.21.15.170:6823","nonce":145026}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6820","nonce":145026},{"type":"v1","addr":"172.21.15.170:6821","nonce":145026}]},"public_addr":"172.21.15.170:6816/145026","cluster_addr":"172.21.15.170:6819/145026","heartbeat_back_addr":"172.21.15.170:6823/145026","heartbeat_front_addr":"172.21.15.170:6821/145026","state":["exists","up"]},{"osd":3,"uuid":"f97e19c6-a762-486b-8b70-47eeb70dfcc3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6824","nonce":145036},{"type":"v1","addr":"172.21.15.170:6825","nonce":145036}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6826","nonce":145036},{"type":"v1","addr":"172.21.15.170:6827","nonce":145036}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6830","nonce":145036},{"type":"v1","addr":"172.21.15.170:6831","nonce":145036}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6828","nonce":145036},{"type":"v1","addr":"172.21.15.170:6829","nonce":145036}]},"public_addr":"172.21.15.170:6825/145036","cluster_addr":"172.21.15.170:6827/145036","heartbeat_back_addr":"172.21.15.170:6831/145036","heartbeat_front_addr":"172.21.15.170:6829/145036","state":["exists","up"]},{"osd":4,"uuid":"3a10cacd-01da-4e94-a001-361dfce676bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":144990},{"type":"v1","addr":"172.21.15.191:6801","nonce":144990}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":144990},{"type":"v1","addr":"172.21.15.191:6803","nonce":144990}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":144990},{"type":"v1","addr":"172.21.15.191:6807","nonce":144990}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":144990},{"type":"v1","addr":"172.21.15.191:6805","nonce":144990}]},"public_addr":"172.21.15.191:6801/144990","cluster_addr":"172.21.15.191:6803/144990","heartbeat_back_addr":"172.21.15.191:6807/144990","heartbeat_front_addr":"172.21.15.191:6805/144990","state":["exists","up"]},{"osd":5,"uuid":"a8bd46a3-dd7f-4df2-a93b-61a6ac38621e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":144991},{"type":"v1","addr":"172.21.15.191:6809","nonce":144991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":144991},{"type":"v1","addr":"172.21.15.191:6811","nonce":144991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":144991},{"type":"v1","addr":"172.21.15.191:6815","nonce":144991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":144991},{"type":"v1","addr":"172.21.15.191:6813","nonce":144991}]},"public_addr":"172.21.15.191:6809/144991","cluster_addr":"172.21.15.191:6811/144991","heartbeat_back_addr":"172.21.15.191:6815/144991","heartbeat_front_addr":"172.21.15.191:6813/144991","state":["exists","up"]},{"osd":6,"uuid":"320479ef-0a92-4b8b-bd4c-2162f512c8c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":145036},{"type":"v1","addr":"172.21.15.191:6825","nonce":145036}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":145036},{"type":"v1","addr":"172.21.15.191:6827","nonce":145036}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":145036},{"type":"v1","addr":"172.21.15.191:6831","nonce":145036}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":145036},{"type":"v1","addr":"172.21.15.191:6829","nonce":145036}]},"public_addr":"172.21.15.191:6825/145036","cluster_addr":"172.21.15.191:6827/145036","heartbeat_back_addr":"172.21.15.191:6831/145036","heartbeat_front_addr":"172.21.15.191:6829/145036","state":["exists","up"]},{"osd":7,"uuid":"7b27eaf0-ac40-41b0-af69-1fc378ddbe64","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":144992},{"type":"v1","addr":"172.21.15.191:6817","nonce":144992}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":144992},{"type":"v1","addr":"172.21.15.191:6819","nonce":144992}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":144992},{"type":"v1","addr":"172.21.15.191:6823","nonce":144992}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":144992},{"type":"v1","addr":"172.21.15.191:6821","nonce":144992}]},"public_addr":"172.21.15.191:6817/144992","cluster_addr":"172.21.15.191:6819/144992","heartbeat_back_addr":"172.21.15.191:6823/144992","heartbeat_front_addr":"172.21.15.191:6821/144992","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.317401+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.607854+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.593088+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.307454+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.354287+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.367514+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.280836+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:02:37.284268+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:02:43.476 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2023-09-22T13:02:43.627 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:43.627 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2023-09-22T13:02:43.977 INFO:teuthology.orchestra.run.smithi170.stdout:60129542147 2023-09-22T13:02:43.977 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2023-09-22T13:02:44.127 INFO:teuthology.orchestra.run.smithi170.stdout:60129542147 2023-09-22T13:02:44.127 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2023-09-22T13:02:44.275 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:44.275 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.4 flush_pg_stats 2023-09-22T13:02:44.426 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:44.426 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.5 flush_pg_stats 2023-09-22T13:02:44.580 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:44.581 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.6 flush_pg_stats 2023-09-22T13:02:44.731 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:44.731 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.7 flush_pg_stats 2023-09-22T13:02:44.882 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:44.882 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2023-09-22T13:02:45.169 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:45.182 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574852 got 55834574852 for osd.0 2023-09-22T13:02:45.182 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2023-09-22T13:02:45.473 INFO:teuthology.orchestra.run.smithi170.stdout:60129542147 2023-09-22T13:02:45.487 INFO:tasks.ceph.ceph_manager.ceph:need seq 60129542147 got 60129542147 for osd.1 2023-09-22T13:02:45.487 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2023-09-22T13:02:45.777 INFO:teuthology.orchestra.run.smithi170.stdout:60129542147 2023-09-22T13:02:45.790 INFO:tasks.ceph.ceph_manager.ceph:need seq 60129542147 got 60129542147 for osd.2 2023-09-22T13:02:45.791 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2023-09-22T13:02:46.077 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:46.091 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574852 got 55834574852 for osd.3 2023-09-22T13:02:46.091 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.4 2023-09-22T13:02:46.380 INFO:teuthology.orchestra.run.smithi170.stdout:55834574851 2023-09-22T13:02:46.393 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574852 got 55834574851 for osd.4 2023-09-22T13:02:47.394 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.4 2023-09-22T13:02:47.697 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:47.711 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574852 got 55834574852 for osd.4 2023-09-22T13:02:47.711 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.5 2023-09-22T13:02:48.001 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:48.015 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574852 got 55834574852 for osd.5 2023-09-22T13:02:48.015 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.6 2023-09-22T13:02:48.303 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:48.317 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574852 got 55834574852 for osd.6 2023-09-22T13:02:48.317 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.7 2023-09-22T13:02:48.611 INFO:teuthology.orchestra.run.smithi170.stdout:55834574852 2023-09-22T13:02:48.625 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574852 got 55834574852 for osd.7 2023-09-22T13:02:48.625 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2023-09-22T13:02:48.625 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2023-09-22T13:02:48.914 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:02:48.916 INFO:teuthology.orchestra.run.smithi170.stderr:dumped all 2023-09-22T13:02:48.927 INFO:teuthology.orchestra.run.smithi170.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2023-09-22T13:02:48.366842+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":4,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":2378072,"kb_used_data":1304,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":752596648,"statfs":{"total":773094113280,"available":770658967552,"internally_reserved":0,"allocated":1335296,"data_stored":435414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":8,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:43.266053+0000","last_change":"2023-09-22T13:02:43.266053+0000","last_active":"2023-09-22T13:02:43.266053+0000","last_peered":"2023-09-22T13:02:43.266053+0000","last_clean":"2023-09-22T13:02:43.266053+0000","last_became_active":"2023-09-22T13:02:43.265715+0000","last_became_peered":"2023-09-22T13:02:43.265715+0000","last_unstale":"2023-09-22T13:02:43.266053+0000","last_undegraded":"2023-09-22T13:02:43.266053+0000","last_fullsized":"2023-09-22T13:02:43.266053+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:43.266819+0000","last_change":"2023-09-22T13:02:43.266172+0000","last_active":"2023-09-22T13:02:43.266819+0000","last_peered":"2023-09-22T13:02:43.266819+0000","last_clean":"2023-09-22T13:02:43.266819+0000","last_became_active":"2023-09-22T13:02:43.265920+0000","last_became_peered":"2023-09-22T13:02:43.265920+0000","last_unstale":"2023-09-22T13:02:43.266819+0000","last_undegraded":"2023-09-22T13:02:43.266819+0000","last_fullsized":"2023-09-22T13:02:43.266819+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":8,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.505539+0000","last_change":"2023-09-22T13:02:42.505539+0000","last_active":"2023-09-22T13:02:42.505539+0000","last_peered":"2023-09-22T13:02:42.505539+0000","last_clean":"2023-09-22T13:02:42.505539+0000","last_became_active":"2023-09-22T13:02:42.505263+0000","last_became_peered":"2023-09-22T13:02:42.505263+0000","last_unstale":"2023-09-22T13:02:42.505539+0000","last_undegraded":"2023-09-22T13:02:42.505539+0000","last_fullsized":"2023-09-22T13:02:42.505539+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":8,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.505594+0000","last_change":"2023-09-22T13:02:42.505594+0000","last_active":"2023-09-22T13:02:42.505594+0000","last_peered":"2023-09-22T13:02:42.505594+0000","last_clean":"2023-09-22T13:02:42.505594+0000","last_became_active":"2023-09-22T13:02:42.505274+0000","last_became_peered":"2023-09-22T13:02:42.505274+0000","last_unstale":"2023-09-22T13:02:42.505594+0000","last_undegraded":"2023-09-22T13:02:42.505594+0000","last_fullsized":"2023-09-22T13:02:42.505594+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":11,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:43.330356+0000","last_change":"2023-09-22T13:02:41.201766+0000","last_active":"2023-09-22T13:02:43.330356+0000","last_peered":"2023-09-22T13:02:43.330356+0000","last_clean":"2023-09-22T13:02:43.330356+0000","last_became_active":"2023-09-22T13:02:41.201486+0000","last_became_peered":"2023-09-22T13:02:41.201486+0000","last_unstale":"2023-09-22T13:02:43.330356+0000","last_undegraded":"2023-09-22T13:02:43.330356+0000","last_fullsized":"2023-09-22T13:02:43.330356+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:39.369147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:39.369147+0000","last_clean_scrub_stamp":"2023-09-22T13:02:39.369147+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":8,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.500818+0000","last_change":"2023-09-22T13:02:42.500818+0000","last_active":"2023-09-22T13:02:42.500818+0000","last_peered":"2023-09-22T13:02:42.500818+0000","last_clean":"2023-09-22T13:02:42.500818+0000","last_became_active":"2023-09-22T13:02:42.500552+0000","last_became_peered":"2023-09-22T13:02:42.500552+0000","last_unstale":"2023-09-22T13:02:42.500818+0000","last_undegraded":"2023-09-22T13:02:42.500818+0000","last_fullsized":"2023-09-22T13:02:42.500818+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.510613+0000","last_change":"2023-09-22T13:02:42.505688+0000","last_active":"2023-09-22T13:02:42.510613+0000","last_peered":"2023-09-22T13:02:42.510613+0000","last_clean":"2023-09-22T13:02:42.510613+0000","last_became_active":"2023-09-22T13:02:42.505327+0000","last_became_peered":"2023-09-22T13:02:42.505327+0000","last_unstale":"2023-09-22T13:02:42.510613+0000","last_undegraded":"2023-09-22T13:02:42.510613+0000","last_fullsized":"2023-09-22T13:02:42.510613+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":8,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.503020+0000","last_change":"2023-09-22T13:02:42.503020+0000","last_active":"2023-09-22T13:02:42.503020+0000","last_peered":"2023-09-22T13:02:42.503020+0000","last_clean":"2023-09-22T13:02:42.503020+0000","last_became_active":"2023-09-22T13:02:42.502719+0000","last_became_peered":"2023-09-22T13:02:42.502719+0000","last_unstale":"2023-09-22T13:02:42.503020+0000","last_undegraded":"2023-09-22T13:02:42.503020+0000","last_fullsized":"2023-09-22T13:02:42.503020+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.510839+0000","last_change":"2023-09-22T13:02:42.508480+0000","last_active":"2023-09-22T13:02:42.510839+0000","last_peered":"2023-09-22T13:02:42.510839+0000","last_clean":"2023-09-22T13:02:42.510839+0000","last_became_active":"2023-09-22T13:02:42.508177+0000","last_became_peered":"2023-09-22T13:02:42.508177+0000","last_unstale":"2023-09-22T13:02:42.510839+0000","last_undegraded":"2023-09-22T13:02:42.510839+0000","last_fullsized":"2023-09-22T13:02:42.510839+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":14,"seq":60129542148,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297268,"kb_used_data":172,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074572,"statfs":{"total":96636764160,"available":96332361728,"internally_reserved":0,"allocated":176128,"data_stored":58780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.254}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32700000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35699999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33900000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48799999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42699999999999999}]}]},{"osd":1,"up_from":14,"seq":60129542148,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297256,"kb_used_data":160,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074584,"statfs":{"total":96636764160,"available":96332374016,"internally_reserved":0,"allocated":163840,"data_stored":52981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28899999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31900000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52300000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.502}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]}]},{"osd":3,"up_from":13,"seq":55834574852,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297256,"kb_used_data":160,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074584,"statfs":{"total":96636764160,"available":96332374016,"internally_reserved":0,"allocated":163840,"data_stored":52981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30199999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.8849999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67200000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55700000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54300000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56899999999999995}]}]},{"osd":0,"up_from":13,"seq":55834574852,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297244,"kb_used_data":148,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074596,"statfs":{"total":96636764160,"available":96332386304,"internally_reserved":0,"allocated":151552,"data_stored":47150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46800000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.432}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41299999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60499999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66300000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63300000000000001}]}]},{"osd":7,"up_from":13,"seq":55834574852,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297256,"kb_used_data":160,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074584,"statfs":{"total":96636764160,"available":96332374016,"internally_reserved":0,"allocated":163840,"data_stored":52981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53000000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70499999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.871}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.99299999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.353}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1970000000000001}]}]},{"osd":4,"up_from":13,"seq":55834574852,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297268,"kb_used_data":172,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074572,"statfs":{"total":96636764160,"available":96332361728,"internally_reserved":0,"allocated":176128,"data_stored":58780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55400000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40699999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80300000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.497}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68700000000000006}]}]},{"osd":5,"up_from":13,"seq":55834574852,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297268,"kb_used_data":172,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074572,"statfs":{"total":96636764160,"available":96332361728,"internally_reserved":0,"allocated":176128,"data_stored":58780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.117}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41399999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.99199999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80800000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.01}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.371}]}]},{"osd":6,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297256,"kb_used_data":160,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074584,"statfs":{"total":96636764160,"available":96332374016,"internally_reserved":0,"allocated":163840,"data_stored":52981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38900000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52300000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54000000000000004}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.625}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-22T13:02:48.929 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2023-09-22T13:02:49.215 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:02:49.217 INFO:teuthology.orchestra.run.smithi170.stderr:dumped all 2023-09-22T13:02:49.228 INFO:teuthology.orchestra.run.smithi170.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2023-09-22T13:02:48.366842+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":4,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":754974720,"kb_used":2378072,"kb_used_data":1304,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":752596648,"statfs":{"total":773094113280,"available":770658967552,"internally_reserved":0,"allocated":1335296,"data_stored":435414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":8,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:43.266053+0000","last_change":"2023-09-22T13:02:43.266053+0000","last_active":"2023-09-22T13:02:43.266053+0000","last_peered":"2023-09-22T13:02:43.266053+0000","last_clean":"2023-09-22T13:02:43.266053+0000","last_became_active":"2023-09-22T13:02:43.265715+0000","last_became_peered":"2023-09-22T13:02:43.265715+0000","last_unstale":"2023-09-22T13:02:43.266053+0000","last_undegraded":"2023-09-22T13:02:43.266053+0000","last_fullsized":"2023-09-22T13:02:43.266053+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:43.266819+0000","last_change":"2023-09-22T13:02:43.266172+0000","last_active":"2023-09-22T13:02:43.266819+0000","last_peered":"2023-09-22T13:02:43.266819+0000","last_clean":"2023-09-22T13:02:43.266819+0000","last_became_active":"2023-09-22T13:02:43.265920+0000","last_became_peered":"2023-09-22T13:02:43.265920+0000","last_unstale":"2023-09-22T13:02:43.266819+0000","last_undegraded":"2023-09-22T13:02:43.266819+0000","last_fullsized":"2023-09-22T13:02:43.266819+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":8,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.505539+0000","last_change":"2023-09-22T13:02:42.505539+0000","last_active":"2023-09-22T13:02:42.505539+0000","last_peered":"2023-09-22T13:02:42.505539+0000","last_clean":"2023-09-22T13:02:42.505539+0000","last_became_active":"2023-09-22T13:02:42.505263+0000","last_became_peered":"2023-09-22T13:02:42.505263+0000","last_unstale":"2023-09-22T13:02:42.505539+0000","last_undegraded":"2023-09-22T13:02:42.505539+0000","last_fullsized":"2023-09-22T13:02:42.505539+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":8,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.505594+0000","last_change":"2023-09-22T13:02:42.505594+0000","last_active":"2023-09-22T13:02:42.505594+0000","last_peered":"2023-09-22T13:02:42.505594+0000","last_clean":"2023-09-22T13:02:42.505594+0000","last_became_active":"2023-09-22T13:02:42.505274+0000","last_became_peered":"2023-09-22T13:02:42.505274+0000","last_unstale":"2023-09-22T13:02:42.505594+0000","last_undegraded":"2023-09-22T13:02:42.505594+0000","last_fullsized":"2023-09-22T13:02:42.505594+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":11,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:43.330356+0000","last_change":"2023-09-22T13:02:41.201766+0000","last_active":"2023-09-22T13:02:43.330356+0000","last_peered":"2023-09-22T13:02:43.330356+0000","last_clean":"2023-09-22T13:02:43.330356+0000","last_became_active":"2023-09-22T13:02:41.201486+0000","last_became_peered":"2023-09-22T13:02:41.201486+0000","last_unstale":"2023-09-22T13:02:43.330356+0000","last_undegraded":"2023-09-22T13:02:43.330356+0000","last_fullsized":"2023-09-22T13:02:43.330356+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:39.369147+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:39.369147+0000","last_clean_scrub_stamp":"2023-09-22T13:02:39.369147+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":8,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.500818+0000","last_change":"2023-09-22T13:02:42.500818+0000","last_active":"2023-09-22T13:02:42.500818+0000","last_peered":"2023-09-22T13:02:42.500818+0000","last_clean":"2023-09-22T13:02:42.500818+0000","last_became_active":"2023-09-22T13:02:42.500552+0000","last_became_peered":"2023-09-22T13:02:42.500552+0000","last_unstale":"2023-09-22T13:02:42.500818+0000","last_undegraded":"2023-09-22T13:02:42.500818+0000","last_fullsized":"2023-09-22T13:02:42.500818+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.510613+0000","last_change":"2023-09-22T13:02:42.505688+0000","last_active":"2023-09-22T13:02:42.510613+0000","last_peered":"2023-09-22T13:02:42.510613+0000","last_clean":"2023-09-22T13:02:42.510613+0000","last_became_active":"2023-09-22T13:02:42.505327+0000","last_became_peered":"2023-09-22T13:02:42.505327+0000","last_unstale":"2023-09-22T13:02:42.510613+0000","last_undegraded":"2023-09-22T13:02:42.510613+0000","last_fullsized":"2023-09-22T13:02:42.510613+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":8,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.503020+0000","last_change":"2023-09-22T13:02:42.503020+0000","last_active":"2023-09-22T13:02:42.503020+0000","last_peered":"2023-09-22T13:02:42.503020+0000","last_clean":"2023-09-22T13:02:42.503020+0000","last_became_active":"2023-09-22T13:02:42.502719+0000","last_became_peered":"2023-09-22T13:02:42.502719+0000","last_unstale":"2023-09-22T13:02:42.503020+0000","last_undegraded":"2023-09-22T13:02:42.503020+0000","last_fullsized":"2023-09-22T13:02:42.503020+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":9,"reported_epoch":17,"state":"active+clean","last_fresh":"2023-09-22T13:02:42.510839+0000","last_change":"2023-09-22T13:02:42.508480+0000","last_active":"2023-09-22T13:02:42.510839+0000","last_peered":"2023-09-22T13:02:42.510839+0000","last_clean":"2023-09-22T13:02:42.510839+0000","last_became_active":"2023-09-22T13:02:42.508177+0000","last_became_peered":"2023-09-22T13:02:42.508177+0000","last_unstale":"2023-09-22T13:02:42.510839+0000","last_undegraded":"2023-09-22T13:02:42.510839+0000","last_fullsized":"2023-09-22T13:02:42.510839+0000","mapping_epoch":16,"log_start":"0'0","ondisk_log_start":"0'0","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:02:41.485510+0000","last_clean_scrub_stamp":"2023-09-22T13:02:41.485510+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":14,"seq":60129542148,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297268,"kb_used_data":172,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074572,"statfs":{"total":96636764160,"available":96332361728,"internally_reserved":0,"allocated":176128,"data_stored":58780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.254}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32700000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35699999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33900000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48799999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42699999999999999}]}]},{"osd":1,"up_from":14,"seq":60129542148,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297256,"kb_used_data":160,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074584,"statfs":{"total":96636764160,"available":96332374016,"internally_reserved":0,"allocated":163840,"data_stored":52981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.22800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28899999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31900000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49099999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52300000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.502}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]}]},{"osd":3,"up_from":13,"seq":55834574852,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297256,"kb_used_data":160,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074584,"statfs":{"total":96636764160,"available":96332374016,"internally_reserved":0,"allocated":163840,"data_stored":52981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30199999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.8849999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67200000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55700000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54300000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56899999999999995}]}]},{"osd":0,"up_from":13,"seq":55834574852,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297244,"kb_used_data":148,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074596,"statfs":{"total":96636764160,"available":96332386304,"internally_reserved":0,"allocated":151552,"data_stored":47150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46800000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.432}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41299999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60499999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66300000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63300000000000001}]}]},{"osd":7,"up_from":13,"seq":55834574852,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297256,"kb_used_data":160,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074584,"statfs":{"total":96636764160,"available":96332374016,"internally_reserved":0,"allocated":163840,"data_stored":52981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53000000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70499999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.871}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.99299999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.353}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.1970000000000001}]}]},{"osd":4,"up_from":13,"seq":55834574852,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297268,"kb_used_data":172,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074572,"statfs":{"total":96636764160,"available":96332361728,"internally_reserved":0,"allocated":176128,"data_stored":58780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55400000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40699999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80300000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.497}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68700000000000006}]}]},{"osd":5,"up_from":13,"seq":55834574852,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297268,"kb_used_data":172,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074572,"statfs":{"total":96636764160,"available":96332361728,"internally_reserved":0,"allocated":176128,"data_stored":58780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.117}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41399999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.99199999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80800000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.01}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.371}]}]},{"osd":6,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":297256,"kb_used_data":160,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":94074584,"statfs":{"total":96636764160,"available":96332374016,"internally_reserved":0,"allocated":163840,"data_stored":52981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38900000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52300000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54000000000000004}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.625}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-22T13:02:49.230 INFO:tasks.ceph.ceph_manager.ceph:clean! 2023-09-22T13:02:49.230 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-22T13:02:49.230 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2023-09-22T13:02:49.231 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2023-09-22T13:02:49.558 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:02:49.572 INFO:teuthology.orchestra.run.smithi170.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-22T13:02:49.572 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2023-09-22T13:02:49.573 INFO:teuthology.run_tasks:Running task openssl_keys... 2023-09-22T13:02:49.584 DEBUG:teuthology.task:Applying overrides for task openssl_keys: {'rgw.client.0': {'ca': 'root', 'client': 'client.0', 'embed-key': True}, 'rgw.client.1': {'ca': 'root', 'client': 'client.1', 'embed-key': True}, 'root': {'client': 'client.0', 'cn': 'teuthology', 'install': ['client.0', 'client.1'], 'key-type': 'rsa:4096'}} 2023-09-22T13:02:49.585 DEBUG:teuthology.orchestra.run.smithi170:> mkdir -p /home/ubuntu/cephtest/ca 2023-09-22T13:02:49.603 DEBUG:teuthology.orchestra.run.smithi170:> openssl req -x509 -nodes -newkey rsa:4096 -keyout /home/ubuntu/cephtest/ca/root.key -days 365 -out /home/ubuntu/cephtest/ca/root.crt -subj /CN=teuthology 2023-09-22T13:02:49.673 INFO:teuthology.orchestra.run.smithi170.stderr:Generating a RSA private key 2023-09-22T13:02:49.798 INFO:teuthology.orchestra.run.smithi170.stderr:......................................++++ 2023-09-22T13:02:49.891 INFO:teuthology.orchestra.run.smithi170.stderr:...........................++++ 2023-09-22T13:02:49.892 INFO:teuthology.orchestra.run.smithi170.stderr:writing new private key to '/home/ubuntu/cephtest/ca/root.key' 2023-09-22T13:02:49.892 INFO:teuthology.orchestra.run.smithi170.stderr:----- 2023-09-22T13:02:49.897 DEBUG:teuthology.orchestra.run.smithi170:> sudo cp /home/ubuntu/cephtest/ca/root.crt /usr/share/pki/ca-trust-source/anchors/root.crt 2023-09-22T13:02:49.971 DEBUG:teuthology.orchestra.run.smithi170:> sudo update-ca-trust 2023-09-22T13:02:50.885 DEBUG:teuthology.orchestra.run.smithi191:> mkdir -p /home/ubuntu/cephtest/ca 2023-09-22T13:02:50.907 INFO:tasks.openssl_keys:copying from ubuntu@smithi170.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/root.crt to ubuntu@smithi191.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/root.crt... 2023-09-22T13:02:50.930 DEBUG:teuthology.orchestra.remote:smithi170:/home/ubuntu/cephtest/ca/root.crt is 2KB 2023-09-22T13:02:51.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo mv /home/ubuntu/cephtest/ca/root.crt /usr/share/pki/ca-trust-source/anchors/root.crt 2023-09-22T13:02:51.051 DEBUG:teuthology.orchestra.run.smithi191:> sudo update-ca-trust 2023-09-22T13:02:51.975 DEBUG:teuthology.orchestra.run.smithi170:> mkdir -p /home/ubuntu/cephtest/ca 2023-09-22T13:02:51.997 DEBUG:teuthology.orchestra.run.smithi170:> openssl req -nodes -newkey rsa:2048 -keyout /home/ubuntu/cephtest/ca/rgw.client.0.key -out /home/ubuntu/cephtest/ca/rgw.client.0.csr -subj /CN=smithi170.front.sepia.ceph.com 2023-09-22T13:02:52.060 INFO:teuthology.orchestra.run.smithi170.stderr:Generating a RSA private key 2023-09-22T13:02:52.112 INFO:teuthology.orchestra.run.smithi170.stderr:....................................................+++++ 2023-09-22T13:02:52.130 INFO:teuthology.orchestra.run.smithi170.stderr:.................+++++ 2023-09-22T13:02:52.130 INFO:teuthology.orchestra.run.smithi170.stderr:writing new private key to '/home/ubuntu/cephtest/ca/rgw.client.0.key' 2023-09-22T13:02:52.130 INFO:teuthology.orchestra.run.smithi170.stderr:----- 2023-09-22T13:02:52.132 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:52.132 DEBUG:teuthology.orchestra.run.smithi170:> dd of=/home/ubuntu/cephtest/ca/rgw.client.0.ext 2023-09-22T13:02:52.191 DEBUG:teuthology.orchestra.run.smithi170:> openssl x509 -req -in /home/ubuntu/cephtest/ca/rgw.client.0.csr -CA /home/ubuntu/cephtest/ca/root.crt -CAkey /home/ubuntu/cephtest/ca/root.key -CAcreateserial -out /home/ubuntu/cephtest/ca/rgw.client.0.crt -days 365 -sha256 -extfile /home/ubuntu/cephtest/ca/rgw.client.0.ext 2023-09-22T13:02:52.256 INFO:teuthology.orchestra.run.smithi170.stderr:Signature ok 2023-09-22T13:02:52.256 INFO:teuthology.orchestra.run.smithi170.stderr:subject=CN = smithi170.front.sepia.ceph.com 2023-09-22T13:02:52.256 INFO:teuthology.orchestra.run.smithi170.stderr:Getting CA Private Key 2023-09-22T13:02:52.262 DEBUG:teuthology.orchestra.run.smithi170:> rm /home/ubuntu/cephtest/ca/rgw.client.0.csr /home/ubuntu/cephtest/ca/root.srl /home/ubuntu/cephtest/ca/rgw.client.0.ext 2023-09-22T13:02:52.321 DEBUG:teuthology.orchestra.run.smithi170:> openssl verify -CAfile /home/ubuntu/cephtest/ca/root.crt /home/ubuntu/cephtest/ca/rgw.client.0.crt 2023-09-22T13:02:52.381 INFO:teuthology.orchestra.run.smithi170.stdout:/home/ubuntu/cephtest/ca/rgw.client.0.crt: OK 2023-09-22T13:02:52.381 DEBUG:teuthology.orchestra.run.smithi170:> cat /home/ubuntu/cephtest/ca/rgw.client.0.key >> /home/ubuntu/cephtest/ca/rgw.client.0.crt 2023-09-22T13:02:52.440 DEBUG:teuthology.orchestra.run.smithi191:> mkdir -p /home/ubuntu/cephtest/ca 2023-09-22T13:02:52.458 DEBUG:teuthology.orchestra.run.smithi170:> openssl req -nodes -newkey rsa:2048 -keyout /home/ubuntu/cephtest/ca/rgw.client.1.key -out /home/ubuntu/cephtest/ca/rgw.client.1.csr -subj /CN=smithi191.front.sepia.ceph.com 2023-09-22T13:02:52.498 INFO:teuthology.orchestra.run.smithi170.stderr:Generating a RSA private key 2023-09-22T13:02:52.534 INFO:teuthology.orchestra.run.smithi170.stderr:..................................+++++ 2023-09-22T13:02:52.559 INFO:teuthology.orchestra.run.smithi170.stderr:........................+++++ 2023-09-22T13:02:52.559 INFO:teuthology.orchestra.run.smithi170.stderr:writing new private key to '/home/ubuntu/cephtest/ca/rgw.client.1.key' 2023-09-22T13:02:52.559 INFO:teuthology.orchestra.run.smithi170.stderr:----- 2023-09-22T13:02:52.561 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:02:52.561 DEBUG:teuthology.orchestra.run.smithi170:> dd of=/home/ubuntu/cephtest/ca/rgw.client.1.ext 2023-09-22T13:02:52.620 DEBUG:teuthology.orchestra.run.smithi170:> openssl x509 -req -in /home/ubuntu/cephtest/ca/rgw.client.1.csr -CA /home/ubuntu/cephtest/ca/root.crt -CAkey /home/ubuntu/cephtest/ca/root.key -CAcreateserial -out /home/ubuntu/cephtest/ca/rgw.client.1.crt -days 365 -sha256 -extfile /home/ubuntu/cephtest/ca/rgw.client.1.ext 2023-09-22T13:02:52.684 INFO:teuthology.orchestra.run.smithi170.stderr:Signature ok 2023-09-22T13:02:52.684 INFO:teuthology.orchestra.run.smithi170.stderr:subject=CN = smithi191.front.sepia.ceph.com 2023-09-22T13:02:52.684 INFO:teuthology.orchestra.run.smithi170.stderr:Getting CA Private Key 2023-09-22T13:02:52.690 DEBUG:teuthology.orchestra.run.smithi170:> rm /home/ubuntu/cephtest/ca/rgw.client.1.csr /home/ubuntu/cephtest/ca/root.srl /home/ubuntu/cephtest/ca/rgw.client.1.ext 2023-09-22T13:02:52.748 DEBUG:teuthology.orchestra.run.smithi170:> openssl verify -CAfile /home/ubuntu/cephtest/ca/root.crt /home/ubuntu/cephtest/ca/rgw.client.1.crt 2023-09-22T13:02:52.807 INFO:teuthology.orchestra.run.smithi170.stdout:/home/ubuntu/cephtest/ca/rgw.client.1.crt: OK 2023-09-22T13:02:52.808 INFO:tasks.openssl_keys:copying from ubuntu@smithi170.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/rgw.client.1.crt to ubuntu@smithi191.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/rgw.client.1.crt... 2023-09-22T13:02:52.868 DEBUG:teuthology.orchestra.remote:smithi170:/home/ubuntu/cephtest/ca/rgw.client.1.crt is 1KB 2023-09-22T13:02:52.939 INFO:tasks.openssl_keys:copying from ubuntu@smithi170.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/rgw.client.1.key to ubuntu@smithi191.front.sepia.ceph.com:/home/ubuntu/cephtest/ca/rgw.client.1.key... 2023-09-22T13:02:52.959 DEBUG:teuthology.orchestra.remote:smithi170:/home/ubuntu/cephtest/ca/rgw.client.1.key is 2KB 2023-09-22T13:02:53.004 DEBUG:teuthology.orchestra.run.smithi170:> rm /home/ubuntu/cephtest/ca/rgw.client.1.crt /home/ubuntu/cephtest/ca/rgw.client.1.key 2023-09-22T13:02:53.022 DEBUG:teuthology.orchestra.run.smithi191:> openssl verify /home/ubuntu/cephtest/ca/rgw.client.1.crt 2023-09-22T13:02:53.069 INFO:teuthology.orchestra.run.smithi191.stdout:/home/ubuntu/cephtest/ca/rgw.client.1.crt: OK 2023-09-22T13:02:53.069 DEBUG:teuthology.orchestra.run.smithi191:> cat /home/ubuntu/cephtest/ca/rgw.client.1.key >> /home/ubuntu/cephtest/ca/rgw.client.1.crt 2023-09-22T13:02:53.088 INFO:teuthology.run_tasks:Running task rgw... 2023-09-22T13:02:53.099 DEBUG:tasks.rgw:config is {'client.0': {'ssl certificate': 'rgw.client.0'}, 'client.1': {'ssl certificate': 'rgw.client.1'}} 2023-09-22T13:02:53.099 DEBUG:tasks.rgw:client list is dict_keys(['client.0', 'client.1']) 2023-09-22T13:02:53.100 INFO:tasks.rgw:Creating data pools 2023-09-22T13:02:53.100 DEBUG:tasks.rgw:Obtaining remote for client client.0 2023-09-22T13:02:53.100 DEBUG:teuthology.orchestra.run.smithi170:> sudo ceph osd erasure-code-profile set client.0 k=2 m=1 crush-failure-domain=osd --cluster ceph 2023-09-22T13:02:53.540 DEBUG:teuthology.orchestra.run.smithi170:> sudo ceph osd pool create default.rgw.buckets.data 64 64 erasure client.0 --cluster ceph 2023-09-22T13:02:55.545 INFO:teuthology.orchestra.run.smithi170.stderr:pool 'default.rgw.buckets.data' created 2023-09-22T13:02:55.565 DEBUG:teuthology.orchestra.run.smithi170:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster ceph 2023-09-22T13:02:56.556 INFO:teuthology.orchestra.run.smithi170.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2023-09-22T13:02:56.575 DEBUG:teuthology.orchestra.run.smithi170:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster ceph 2023-09-22T13:02:57.552 INFO:teuthology.orchestra.run.smithi170.stderr:pool 'default.rgw.buckets.index' created 2023-09-22T13:02:57.564 DEBUG:teuthology.orchestra.run.smithi170:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster ceph 2023-09-22T13:02:58.551 INFO:teuthology.orchestra.run.smithi170.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2023-09-22T13:02:58.572 DEBUG:tasks.rgw:Obtaining remote for client client.1 2023-09-22T13:02:58.572 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph osd erasure-code-profile set client.1 k=2 m=1 crush-failure-domain=osd --cluster ceph 2023-09-22T13:02:59.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph osd pool create default.rgw.buckets.data 64 64 erasure client.1 --cluster ceph 2023-09-22T13:02:59.853 INFO:teuthology.orchestra.run.smithi191.stderr:pool 'default.rgw.buckets.data' already exists 2023-09-22T13:02:59.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster ceph 2023-09-22T13:03:00.587 INFO:teuthology.orchestra.run.smithi191.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2023-09-22T13:03:00.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster ceph 2023-09-22T13:03:00.889 INFO:teuthology.orchestra.run.smithi191.stderr:pool 'default.rgw.buckets.index' already exists 2023-09-22T13:03:00.900 DEBUG:teuthology.orchestra.run.smithi191:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster ceph 2023-09-22T13:03:01.587 INFO:teuthology.orchestra.run.smithi191.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2023-09-22T13:03:01.598 DEBUG:tasks.rgw:Pools created 2023-09-22T13:03:01.598 INFO:tasks.rgw:Configuring compression type = random 2023-09-22T13:03:01.598 INFO:tasks.util.rgw:rgwadmin: client.0 : ['user', 'list'] 2023-09-22T13:03:01.598 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'user', 'list'] 2023-09-22T13:03:01.599 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph user list 2023-09-22T13:03:01.784 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:01.784 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:01.791 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:01.791+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.608 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.607+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.608 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.607+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.609 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.608+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.609 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.608+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.609 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.608+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.610 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.609+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.610 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.609+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.622 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.621+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.622 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.621+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.622 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.621+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.622 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.621+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.623 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.621+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.623 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.621+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.623 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.621+0000 7f69113bc500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:03:03.623 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.621+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.623 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.622+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.623 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.622+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.624 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.622+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.624 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.622+0000 7f69113bc500 10 Creating default zonegroup 2023-09-22T13:03:03.624 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.622+0000 7f69113bc500 20 get_system_obj_state: rctx=0x7ffc56832e10 obj=:.pools.avail state=0x55c6cdb9d810 s->prefetch_data=0 2023-09-22T13:03:03.624 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.622+0000 7f69113bc500 10 couldn't find old data placement pools config, setting up new ones for the zone 2023-09-22T13:03:03.627 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.627+0000 7f68fd09f700 0 -- 172.21.15.170:0/696717107 >> [v2:172.21.15.191:6824/145036,v1:172.21.15.191:6825/145036] conn(0x55c6cdcdb170 msgr2=0x55c6cdcdd560 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:03:03.632 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.631+0000 7f69113bc500 10 WARNING: store->list_zones() returned r=-2 2023-09-22T13:03:03.632 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.631+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.632 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.632+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.635+0000 7f68f7fff700 0 -- 172.21.15.170:0/696717107 >> [v2:172.21.15.170:6800/145025,v1:172.21.15.170:6801/145025] conn(0x55c6cdce2420 msgr2=0x55c6cdce4810 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:03:03.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.635+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.635+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.635+0000 7f69113bc500 10 could not read realm id: (2) No such file or directory 2023-09-22T13:03:03.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.635+0000 7f69113bc500 10 WARNING: failed to set zone as default, r=-22 2023-09-22T13:03:03.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.635+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.635+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.638 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.638+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.639 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.638+0000 7f69113bc500 20 rados_obj.operate() r=0 bl.length=873 2023-09-22T13:03:03.639 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.638+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.640 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.639+0000 7f69113bc500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:03:03.640 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.639+0000 7f69113bc500 20 zonegroup default 2023-09-22T13:03:03.640 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.639+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.640 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.640+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:03.641 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.640+0000 7f69113bc500 10 Cannot find current period zone using local zone 2023-09-22T13:03:03.641 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.640+0000 7f69113bc500 10 Using default name default 2023-09-22T13:03:03.641 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.640+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.641 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.640+0000 7f69113bc500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:03.642 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.640+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:03.642 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.640+0000 7f69113bc500 20 rados_obj.operate() r=0 bl.length=873 2023-09-22T13:03:03.642 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.640+0000 7f69113bc500 20 zone default found 2023-09-22T13:03:03.642 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:03.640+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:05.624 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:05.623+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:05.624 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:05.623+0000 7f69113bc500 20 rados->read ofs=0 len=0 2023-09-22T13:03:05.625 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:05.625+0000 7f69113bc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:05.626 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:05.625+0000 7f69113bc500 20 started sync module instance, tier type = 2023-09-22T13:03:05.626 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:05.625+0000 7f69113bc500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:03:07.643 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.643+0000 7f69113bc500 20 add_watcher() i=0 2023-09-22T13:03:07.644 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.643+0000 7f69113bc500 20 add_watcher() i=1 2023-09-22T13:03:07.644 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.643+0000 7f69113bc500 20 add_watcher() i=2 2023-09-22T13:03:07.644 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.643+0000 7f69113bc500 20 add_watcher() i=3 2023-09-22T13:03:07.644 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.643+0000 7f69113bc500 20 add_watcher() i=4 2023-09-22T13:03:07.644 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.643+0000 7f69113bc500 20 add_watcher() i=5 2023-09-22T13:03:07.644 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.643+0000 7f69113bc500 20 add_watcher() i=6 2023-09-22T13:03:07.644 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.644+0000 7f69113bc500 20 add_watcher() i=7 2023-09-22T13:03:07.645 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.644+0000 7f69113bc500 2 all 8 watchers are set, enabling cache 2023-09-22T13:03:07.645 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.644+0000 7f69113bc500 5 tl::expected, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):409: oid=data_loggenerations_metadata not found 2023-09-22T13:03:07.645 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.644+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.0 2023-09-22T13:03:07.645 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.644+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.645 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.645+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.0 does not exist 2023-09-22T13:03:07.645 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.645+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.1 2023-09-22T13:03:07.646 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.645+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.645+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.1 does not exist 2023-09-22T13:03:07.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.645+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.2 2023-09-22T13:03:07.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.645+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.645+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.2 does not exist 2023-09-22T13:03:07.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.645+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.3 2023-09-22T13:03:07.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.645+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.3 does not exist 2023-09-22T13:03:07.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.4 2023-09-22T13:03:07.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.4 does not exist 2023-09-22T13:03:07.648 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.5 2023-09-22T13:03:07.648 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.648 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.5 does not exist 2023-09-22T13:03:07.648 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.6 2023-09-22T13:03:07.648 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.650 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.6 does not exist 2023-09-22T13:03:07.650 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.7 2023-09-22T13:03:07.650 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.646+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.651 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.7 does not exist 2023-09-22T13:03:07.651 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.8 2023-09-22T13:03:07.651 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.651 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.8 does not exist 2023-09-22T13:03:07.651 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.9 2023-09-22T13:03:07.651 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.651 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.9 does not exist 2023-09-22T13:03:07.651 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.10 2023-09-22T13:03:07.651 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.652 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.10 does not exist 2023-09-22T13:03:07.652 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.11 2023-09-22T13:03:07.652 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.647+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.652 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.648+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.11 does not exist 2023-09-22T13:03:07.652 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.648+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.12 2023-09-22T13:03:07.652 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.648+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.652 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.648+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.12 does not exist 2023-09-22T13:03:07.652 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.648+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.13 2023-09-22T13:03:07.652 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.648+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.652 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.648+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.13 does not exist 2023-09-22T13:03:07.653 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.648+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.14 2023-09-22T13:03:07.653 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.648+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.653 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.14 does not exist 2023-09-22T13:03:07.653 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.15 2023-09-22T13:03:07.653 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.653 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.15 does not exist 2023-09-22T13:03:07.653 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.16 2023-09-22T13:03:07.653 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.653 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.16 does not exist 2023-09-22T13:03:07.653 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.17 2023-09-22T13:03:07.653 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.654 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.17 does not exist 2023-09-22T13:03:07.654 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.18 2023-09-22T13:03:07.654 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.649+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.654 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.18 does not exist 2023-09-22T13:03:07.654 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.19 2023-09-22T13:03:07.654 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.657 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.19 does not exist 2023-09-22T13:03:07.657 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.20 2023-09-22T13:03:07.657 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.657 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.20 does not exist 2023-09-22T13:03:07.657 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.21 2023-09-22T13:03:07.658 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.658 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.21 does not exist 2023-09-22T13:03:07.658 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.22 2023-09-22T13:03:07.658 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.650+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.658 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.651+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.22 does not exist 2023-09-22T13:03:07.658 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.651+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.23 2023-09-22T13:03:07.658 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.651+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.658 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.651+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.23 does not exist 2023-09-22T13:03:07.658 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.651+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.24 2023-09-22T13:03:07.658 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.651+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.658 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.651+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.24 does not exist 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.651+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.25 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.651+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.25 does not exist 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.26 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.26 does not exist 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.27 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.27 does not exist 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.28 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.659 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.28 does not exist 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.29 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.652+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.653+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.29 does not exist 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.653+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.30 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.653+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.653+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.30 does not exist 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.653+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.31 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.653+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.653+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.31 does not exist 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.653+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.32 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.653+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.660 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.654+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.32 does not exist 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.654+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.33 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.654+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.654+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.33 does not exist 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.654+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.34 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.654+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.654+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.34 does not exist 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.654+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.35 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.654+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.655+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.35 does not exist 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.655+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.36 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.655+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.661 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.655+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.36 does not exist 2023-09-22T13:03:07.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.655+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.37 2023-09-22T13:03:07.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.655+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.655+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.37 does not exist 2023-09-22T13:03:07.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.655+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.38 2023-09-22T13:03:07.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.655+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.38 does not exist 2023-09-22T13:03:07.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.39 2023-09-22T13:03:07.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.39 does not exist 2023-09-22T13:03:07.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.40 2023-09-22T13:03:07.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.663 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.40 does not exist 2023-09-22T13:03:07.663 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.41 2023-09-22T13:03:07.663 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.663 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.41 does not exist 2023-09-22T13:03:07.663 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.42 2023-09-22T13:03:07.663 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.656+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.668 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.657+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.42 does not exist 2023-09-22T13:03:07.668 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.657+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.43 2023-09-22T13:03:07.668 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.657+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.657+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.43 does not exist 2023-09-22T13:03:07.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.657+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.44 2023-09-22T13:03:07.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.657+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.657+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.44 does not exist 2023-09-22T13:03:07.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.657+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.45 2023-09-22T13:03:07.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.657+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.658+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.45 does not exist 2023-09-22T13:03:07.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.658+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.46 2023-09-22T13:03:07.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.658+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.658+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.46 does not exist 2023-09-22T13:03:07.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.658+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.47 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.658+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.47 does not exist 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.48 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.48 does not exist 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.49 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.49 does not exist 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.50 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.50 does not exist 2023-09-22T13:03:07.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.51 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.659+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.51 does not exist 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.52 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.52 does not exist 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.53 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.53 does not exist 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.54 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.54 does not exist 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.55 2023-09-22T13:03:07.671 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.660+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.661+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.55 does not exist 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.661+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.56 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.661+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.661+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.56 does not exist 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.661+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.57 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.661+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.661+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.57 does not exist 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.661+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.58 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.661+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.58 does not exist 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.59 2023-09-22T13:03:07.672 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.59 does not exist 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.60 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.60 does not exist 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.61 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.61 does not exist 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.62 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.662+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.62 does not exist 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.63 2023-09-22T13:03:07.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.674 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.63 does not exist 2023-09-22T13:03:07.674 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.64 2023-09-22T13:03:07.674 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.674 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.64 does not exist 2023-09-22T13:03:07.674 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.65 2023-09-22T13:03:07.674 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.674 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.65 does not exist 2023-09-22T13:03:07.674 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.66 2023-09-22T13:03:07.674 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.663+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.674 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.66 does not exist 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.67 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.67 does not exist 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.68 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.68 does not exist 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.69 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.69 does not exist 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.70 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.664+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.675 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.70 does not exist 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.71 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.71 does not exist 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.72 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.72 does not exist 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.73 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.73 does not exist 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.74 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.74 does not exist 2023-09-22T13:03:07.676 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.75 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.665+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.666+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.75 does not exist 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.666+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.76 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.666+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.666+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.76 does not exist 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.666+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.77 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.666+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.666+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.77 does not exist 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.666+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.78 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.666+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.78 does not exist 2023-09-22T13:03:07.677 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.79 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.79 does not exist 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.80 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.80 does not exist 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.81 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.81 does not exist 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.82 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.667+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.82 does not exist 2023-09-22T13:03:07.678 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.83 2023-09-22T13:03:07.679 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.679 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.83 does not exist 2023-09-22T13:03:07.679 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.84 2023-09-22T13:03:07.679 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.683 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.84 does not exist 2023-09-22T13:03:07.684 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.85 2023-09-22T13:03:07.684 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.684 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.85 does not exist 2023-09-22T13:03:07.684 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.86 2023-09-22T13:03:07.684 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.668+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.684 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.86 does not exist 2023-09-22T13:03:07.684 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.87 2023-09-22T13:03:07.684 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.684 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.87 does not exist 2023-09-22T13:03:07.684 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.88 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.88 does not exist 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.89 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.89 does not exist 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.90 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.669+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.670+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.90 does not exist 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.670+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.91 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.670+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.670+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.91 does not exist 2023-09-22T13:03:07.685 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.670+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.92 2023-09-22T13:03:07.686 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.670+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.686 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.670+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.92 does not exist 2023-09-22T13:03:07.686 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.670+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.93 2023-09-22T13:03:07.686 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.670+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.686 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.93 does not exist 2023-09-22T13:03:07.686 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.94 2023-09-22T13:03:07.686 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.686 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.94 does not exist 2023-09-22T13:03:07.686 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.95 2023-09-22T13:03:07.686 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.686 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.95 does not exist 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.96 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.96 does not exist 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.97 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.671+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.97 does not exist 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.98 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.98 does not exist 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.99 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.687 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.99 does not exist 2023-09-22T13:03:07.688 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.100 2023-09-22T13:03:07.688 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.688 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.100 does not exist 2023-09-22T13:03:07.688 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.101 2023-09-22T13:03:07.688 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.672+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.688 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.101 does not exist 2023-09-22T13:03:07.688 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.102 2023-09-22T13:03:07.688 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.688 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.102 does not exist 2023-09-22T13:03:07.688 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.103 2023-09-22T13:03:07.688 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.103 does not exist 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.104 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.104 does not exist 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.105 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.673+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.674+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.105 does not exist 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.674+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.106 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.674+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.674+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.106 does not exist 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.674+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.107 2023-09-22T13:03:07.689 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.674+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.690 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.674+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.107 does not exist 2023-09-22T13:03:07.690 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.674+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.108 2023-09-22T13:03:07.690 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.674+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.690 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.108 does not exist 2023-09-22T13:03:07.690 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.109 2023-09-22T13:03:07.690 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.690 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.109 does not exist 2023-09-22T13:03:07.690 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.110 2023-09-22T13:03:07.690 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.690 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.110 does not exist 2023-09-22T13:03:07.690 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.111 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.111 does not exist 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.112 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.675+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.112 does not exist 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.113 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.113 does not exist 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.114 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.114 does not exist 2023-09-22T13:03:07.691 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.115 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.115 does not exist 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.116 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.676+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.677+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.116 does not exist 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.677+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.117 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.677+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.677+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.117 does not exist 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.677+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.118 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.677+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.677+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.118 does not exist 2023-09-22T13:03:07.692 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.677+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.119 2023-09-22T13:03:07.693 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.677+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.693 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.119 does not exist 2023-09-22T13:03:07.693 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.120 2023-09-22T13:03:07.693 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.693 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.120 does not exist 2023-09-22T13:03:07.693 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.121 2023-09-22T13:03:07.693 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.693 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.121 does not exist 2023-09-22T13:03:07.693 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.122 2023-09-22T13:03:07.693 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.693 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.122 does not exist 2023-09-22T13:03:07.694 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.123 2023-09-22T13:03:07.694 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.678+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.694 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.679+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.123 does not exist 2023-09-22T13:03:07.694 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.679+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.124 2023-09-22T13:03:07.694 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.679+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.694 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.679+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.124 does not exist 2023-09-22T13:03:07.694 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.679+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.125 2023-09-22T13:03:07.694 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.679+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.694 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.679+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.125 does not exist 2023-09-22T13:03:07.694 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.679+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.126 2023-09-22T13:03:07.694 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.679+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.695 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.680+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.126 does not exist 2023-09-22T13:03:07.695 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.680+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.127 2023-09-22T13:03:07.695 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.680+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.695 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.680+0000 7f69113bc500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.127 does not exist 2023-09-22T13:03:07.695 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.680+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::create(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, std::optional >, bool, uint64_t, uint64_t):1206 entering 2023-09-22T13:03:07.695 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.681+0000 7f69113bc500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:03:07.695 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.685+0000 7f68cd7ea700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:03:07.695 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.685+0000 7f69113bc500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:07.696 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:07.685+0000 7f69113bc500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:10.632 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.629+0000 7f69113bc500 10 rgw_init_ioctx warning: failed to set recovery_priority on default.rgw.meta 2023-09-22T13:03:10.632 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.629+0000 7f69113bc500 5 note: GC not initialized 2023-09-22T13:03:10.632 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.631+0000 7f682ffff700 20 reqs_thread_entry: start 2023-09-22T13:03:10.632 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.631+0000 7f682dffb700 20 reqs_thread_entry: start 2023-09-22T13:03:10.632 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.631+0000 7f69113bc500 20 init_complete bucket index max shards: 11 2023-09-22T13:03:10.632 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.631+0000 7f69113bc500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:03:10.632 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.631+0000 7f681ffff700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:03:40 2023 2023-09-22T13:03:10.632 INFO:teuthology.orchestra.run.smithi170.stderr: 2023-09-22T13:03:10.633 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.631+0000 7f681f7fe700 20 reqs_thread_entry: start 2023-09-22T13:03:10.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.662+0000 7f69113bc500 20 remove_watcher() i=0 2023-09-22T13:03:10.662 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.662+0000 7f69113bc500 2 removed watcher, disabling cache 2023-09-22T13:03:10.664 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.663+0000 7f69113bc500 20 remove_watcher() i=1 2023-09-22T13:03:10.665 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.664+0000 7f69113bc500 20 remove_watcher() i=2 2023-09-22T13:03:10.667 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.666+0000 7f69113bc500 20 remove_watcher() i=3 2023-09-22T13:03:10.667 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.667+0000 7f69113bc500 20 remove_watcher() i=4 2023-09-22T13:03:10.669 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.668+0000 7f69113bc500 20 remove_watcher() i=5 2023-09-22T13:03:10.670 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.670+0000 7f69113bc500 20 remove_watcher() i=6 2023-09-22T13:03:10.673 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.672+0000 7f69113bc500 20 remove_watcher() i=7 2023-09-22T13:03:10.674 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.673+0000 7f68cd7ea700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:03:10.677 INFO:teuthology.orchestra.run.smithi170.stdout:[] 2023-09-22T13:03:10.679 DEBUG:tasks.util.rgw: json result: [] 2023-09-22T13:03:10.680 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2023-09-22T13:03:10.680 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2023-09-22T13:03:10.680 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zone placement modify --rgw-zone default --placement-id default-placement --compression random 2023-09-22T13:03:10.729 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:10.729 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:10.735 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.735+0000 7f761cc82500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:10.736 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.735+0000 7f761cc82500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:10.740 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.739+0000 7f7551ffb700 20 reqs_thread_entry: start 2023-09-22T13:03:10.740 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.739+0000 7f761cc82500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.740 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.740+0000 7f761cc82500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:10.741 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.740+0000 7f761cc82500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.741 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.741+0000 7f761cc82500 20 rados_obj.operate() r=0 bl.length=873 2023-09-22T13:03:10.741 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.741+0000 7f761cc82500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.742 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.742+0000 7f761cc82500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:10.743 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.742+0000 7f761cc82500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.743 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.742+0000 7f761cc82500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:10.743 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.742+0000 7f761cc82500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.743 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:10.743+0000 7f761cc82500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:03:10.748 INFO:teuthology.orchestra.run.smithi170.stdout:{ 2023-09-22T13:03:10.748 INFO:teuthology.orchestra.run.smithi170.stdout: "id": "772ce3cc-30ad-4ea4-82fa-9614e9a1eee1", 2023-09-22T13:03:10.748 INFO:teuthology.orchestra.run.smithi170.stdout: "name": "default", 2023-09-22T13:03:10.748 INFO:teuthology.orchestra.run.smithi170.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:03:10.748 INFO:teuthology.orchestra.run.smithi170.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:03:10.748 INFO:teuthology.orchestra.run.smithi170.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:03:10.748 INFO:teuthology.orchestra.run.smithi170.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:03:10.748 INFO:teuthology.orchestra.run.smithi170.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:03:10.748 INFO:teuthology.orchestra.run.smithi170.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: "system_key": { 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: "access_key": "", 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: "secret_key": "" 2023-09-22T13:03:10.749 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: "placement_pools": [ 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: { 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: "key": "default-placement", 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: "val": { 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: "storage_classes": { 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: "STANDARD": { 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: "compression_type": "random" 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:03:10.750 INFO:teuthology.orchestra.run.smithi170.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:03:10.751 INFO:teuthology.orchestra.run.smithi170.stdout: "index_type": 0 2023-09-22T13:03:10.751 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:10.751 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:10.751 INFO:teuthology.orchestra.run.smithi170.stdout: ], 2023-09-22T13:03:10.751 INFO:teuthology.orchestra.run.smithi170.stdout: "realm_id": "", 2023-09-22T13:03:10.751 INFO:teuthology.orchestra.run.smithi170.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:03:10.751 INFO:teuthology.orchestra.run.smithi170.stdout:} 2023-09-22T13:03:10.752 DEBUG:tasks.util.rgw: json result: {'id': '772ce3cc-30ad-4ea4-82fa-9614e9a1eee1', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:03:10.752 INFO:tasks.util.rgw:rgwadmin: client.1 : ['user', 'list'] 2023-09-22T13:03:10.752 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'user', 'list'] 2023-09-22T13:03:10.752 DEBUG:teuthology.orchestra.run.smithi191:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph user list 2023-09-22T13:03:10.876 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:10.876 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:10.884 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.882+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.886 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.884+0000 7fa87918b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:10.886 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.884+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.887 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.885+0000 7fa87918b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:10.887 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.885+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.887 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.886+0000 7fa87918b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:10.887 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.886+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.887 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.886+0000 7fa87918b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:10.900 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.898+0000 7fa87918b500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:03:10.900 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.898+0000 7fa87918b500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:03:10.900 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.898+0000 7fa87918b500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:03:10.900 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.898+0000 7fa87918b500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:03:10.900 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.898+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.900 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.898+0000 7fa87918b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:10.900 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.898+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.901 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.899+0000 7fa87918b500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:03:10.901 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.899+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.901 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.899+0000 7fa87918b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:10.901 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.899+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.901 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.900+0000 7fa87918b500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:03:10.901 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.900+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.901 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.900+0000 7fa87918b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:10.901 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.900+0000 7fa87918b500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:03:10.902 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.900+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.902 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.900+0000 7fa87918b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.900+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.901+0000 7fa87918b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.901+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.901+0000 7fa87918b500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:03:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.901+0000 7fa87918b500 20 zonegroup default 2023-09-22T13:03:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.901+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.901+0000 7fa87918b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.901+0000 7fa87918b500 10 Cannot find current period zone using local zone 2023-09-22T13:03:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.901+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.902+0000 7fa87918b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.902+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.904 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.902+0000 7fa87918b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:10.904 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.902+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.904 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.902+0000 7fa87918b500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:03:10.905 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.902+0000 7fa87918b500 20 zone default found 2023-09-22T13:03:10.905 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.902+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.905 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.903+0000 7fa87918b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:10.905 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.903+0000 7fa87918b500 20 rados->read ofs=0 len=0 2023-09-22T13:03:10.905 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.903+0000 7fa87918b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:10.905 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.903+0000 7fa87918b500 20 started sync module instance, tier type = 2023-09-22T13:03:10.905 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.903+0000 7fa87918b500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:03:10.916 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.915+0000 7fa863e6c700 0 -- 172.21.15.191:0/4080899581 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7fa8540073f0 msgr2=0x7fa854007830 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:03:10.917 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.916+0000 7fa87918b500 20 add_watcher() i=0 2023-09-22T13:03:10.917 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.916+0000 7fa87918b500 20 add_watcher() i=1 2023-09-22T13:03:10.917 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.916+0000 7fa87918b500 20 add_watcher() i=2 2023-09-22T13:03:10.918 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.916+0000 7fa87918b500 20 add_watcher() i=3 2023-09-22T13:03:10.918 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.916+0000 7fa87918b500 20 add_watcher() i=4 2023-09-22T13:03:10.918 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.916+0000 7fa87918b500 20 add_watcher() i=5 2023-09-22T13:03:10.918 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.916+0000 7fa87918b500 20 add_watcher() i=6 2023-09-22T13:03:10.919 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.918+0000 7fa87918b500 20 add_watcher() i=7 2023-09-22T13:03:10.919 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.918+0000 7fa87918b500 2 all 8 watchers are set, enabling cache 2023-09-22T13:03:10.921 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.919+0000 7fa835fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:03:10.921 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.920+0000 7fa87918b500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:10.921 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.920+0000 7fa87918b500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:10.923 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.922+0000 7fa864e6e700 0 -- 172.21.15.191:0/2394217919 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x5642f35e18b0 msgr2=0x5642f35e7d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:03:10.926 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.925+0000 7fa87918b500 5 note: GC not initialized 2023-09-22T13:03:10.927 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.926+0000 7fa7acff9700 20 reqs_thread_entry: start 2023-09-22T13:03:10.928 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.926+0000 7fa7a27fc700 20 reqs_thread_entry: start 2023-09-22T13:03:10.928 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.926+0000 7fa87918b500 20 init_complete bucket index max shards: 11 2023-09-22T13:03:10.928 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.926+0000 7fa87918b500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:03:10.928 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.926+0000 7fa7a0ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:03:41 2023 2023-09-22T13:03:10.928 INFO:teuthology.orchestra.run.smithi191.stderr: 2023-09-22T13:03:10.928 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.926+0000 7fa787fff700 20 reqs_thread_entry: start 2023-09-22T13:03:10.939 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.938+0000 7fa87918b500 20 remove_watcher() i=0 2023-09-22T13:03:10.939 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.938+0000 7fa87918b500 2 removed watcher, disabling cache 2023-09-22T13:03:10.940 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.939+0000 7fa87918b500 20 remove_watcher() i=1 2023-09-22T13:03:10.941 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.940+0000 7fa87918b500 20 remove_watcher() i=2 2023-09-22T13:03:10.942 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.941+0000 7fa87918b500 20 remove_watcher() i=3 2023-09-22T13:03:10.943 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.942+0000 7fa87918b500 20 remove_watcher() i=4 2023-09-22T13:03:10.945 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.944+0000 7fa87918b500 20 remove_watcher() i=5 2023-09-22T13:03:10.946 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.945+0000 7fa87918b500 20 remove_watcher() i=6 2023-09-22T13:03:10.947 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.946+0000 7fa87918b500 20 remove_watcher() i=7 2023-09-22T13:03:10.947 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:10.946+0000 7fa835fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:03:10.951 INFO:teuthology.orchestra.run.smithi191.stdout:[] 2023-09-22T13:03:10.952 DEBUG:tasks.util.rgw: json result: [] 2023-09-22T13:03:10.953 INFO:tasks.util.rgw:rgwadmin: client.1 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2023-09-22T13:03:10.953 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2023-09-22T13:03:10.953 DEBUG:teuthology.orchestra.run.smithi191:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph zone placement modify --rgw-zone default --placement-id default-placement --compression random 2023-09-22T13:03:10.999 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:11.001 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:11.006 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.005+0000 7fb02dcc4500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:11.006 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.005+0000 7fb02dcc4500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:11.010 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.009+0000 7faf627fc700 20 reqs_thread_entry: start 2023-09-22T13:03:11.010 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.009+0000 7fb02dcc4500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.011 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.010+0000 7fb02dcc4500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.011 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.010+0000 7fb02dcc4500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.012 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.011+0000 7fb02dcc4500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:03:11.012 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.011+0000 7fb02dcc4500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.012 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.011+0000 7fb02dcc4500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.013 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.011+0000 7fb02dcc4500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.013 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.011+0000 7fb02dcc4500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.013 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.011+0000 7fb02dcc4500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.013 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.012+0000 7fb02dcc4500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:03:11.018 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-09-22T13:03:11.018 INFO:teuthology.orchestra.run.smithi191.stdout: "id": "772ce3cc-30ad-4ea4-82fa-9614e9a1eee1", 2023-09-22T13:03:11.018 INFO:teuthology.orchestra.run.smithi191.stdout: "name": "default", 2023-09-22T13:03:11.018 INFO:teuthology.orchestra.run.smithi191.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:03:11.018 INFO:teuthology.orchestra.run.smithi191.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:03:11.018 INFO:teuthology.orchestra.run.smithi191.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "system_key": { 2023-09-22T13:03:11.019 INFO:teuthology.orchestra.run.smithi191.stdout: "access_key": "", 2023-09-22T13:03:11.020 INFO:teuthology.orchestra.run.smithi191.stdout: "secret_key": "" 2023-09-22T13:03:11.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-09-22T13:03:11.020 INFO:teuthology.orchestra.run.smithi191.stdout: "placement_pools": [ 2023-09-22T13:03:11.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-09-22T13:03:11.020 INFO:teuthology.orchestra.run.smithi191.stdout: "key": "default-placement", 2023-09-22T13:03:11.020 INFO:teuthology.orchestra.run.smithi191.stdout: "val": { 2023-09-22T13:03:11.020 INFO:teuthology.orchestra.run.smithi191.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:03:11.020 INFO:teuthology.orchestra.run.smithi191.stdout: "storage_classes": { 2023-09-22T13:03:11.020 INFO:teuthology.orchestra.run.smithi191.stdout: "STANDARD": { 2023-09-22T13:03:11.020 INFO:teuthology.orchestra.run.smithi191.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:03:11.020 INFO:teuthology.orchestra.run.smithi191.stdout: "compression_type": "random" 2023-09-22T13:03:11.021 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-09-22T13:03:11.021 INFO:teuthology.orchestra.run.smithi191.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:03:11.021 INFO:teuthology.orchestra.run.smithi191.stdout: "index_type": 0 2023-09-22T13:03:11.021 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.021 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.021 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-09-22T13:03:11.021 INFO:teuthology.orchestra.run.smithi191.stdout: "realm_id": "", 2023-09-22T13:03:11.021 INFO:teuthology.orchestra.run.smithi191.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:03:11.021 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-09-22T13:03:11.022 DEBUG:tasks.util.rgw: json result: {'id': '772ce3cc-30ad-4ea4-82fa-9614e9a1eee1', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:03:11.022 INFO:tasks.util.rgw:rgwadmin: client.0 : ['user', 'list'] 2023-09-22T13:03:11.022 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'user', 'list'] 2023-09-22T13:03:11.022 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph user list 2023-09-22T13:03:11.063 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:11.063 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:11.069 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.068+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.071 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.070+0000 7f5a63560500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.071 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.070+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.072 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.071+0000 7f5a63560500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.072 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.071+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.072 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.071+0000 7f5a63560500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.072 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.071+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.073 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.072+0000 7f5a63560500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.082 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.082+0000 7f5a63560500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:03:11.082 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.082+0000 7f5a63560500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:03:11.083 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.082+0000 7f5a63560500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:03:11.083 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.082+0000 7f5a63560500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:03:11.083 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.082+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.083 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.082+0000 7f5a63560500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.083 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.082+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.084 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.082+0000 7f5a63560500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:03:11.084 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.082+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.084 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.082+0000 7f5a63560500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.084 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.082+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.084 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.083+0000 7f5a63560500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:03:11.085 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.083+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.085 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.083+0000 7f5a63560500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.085 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.083+0000 7f5a63560500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:03:11.085 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.083+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.085 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.083+0000 7f5a63560500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.085 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.083+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.085 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.085 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.086 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:03:11.087 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 20 zonegroup default 2023-09-22T13:03:11.087 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.087 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.087 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 10 Cannot find current period zone using local zone 2023-09-22T13:03:11.087 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.087 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.087 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.087 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.087 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.084+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.087 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.085+0000 7f5a63560500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:03:11.087 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.085+0000 7f5a63560500 20 zone default found 2023-09-22T13:03:11.088 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.085+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.088 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.085+0000 7f5a63560500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.088 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.085+0000 7f5a63560500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.088 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.085+0000 7f5a63560500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.088 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.085+0000 7f5a63560500 20 started sync module instance, tier type = 2023-09-22T13:03:11.088 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.085+0000 7f5a63560500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:03:11.095 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.094+0000 7f5a4e241700 0 -- 172.21.15.170:0/2260257667 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f5a44014710 msgr2=0x7f5a440208f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:03:11.095 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.094+0000 7f5a63560500 20 add_watcher() i=0 2023-09-22T13:03:11.095 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.094+0000 7f5a63560500 20 add_watcher() i=1 2023-09-22T13:03:11.095 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.094+0000 7f5a63560500 20 add_watcher() i=2 2023-09-22T13:03:11.095 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.094+0000 7f5a63560500 20 add_watcher() i=3 2023-09-22T13:03:11.095 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.094+0000 7f5a63560500 20 add_watcher() i=4 2023-09-22T13:03:11.096 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.094+0000 7f5a63560500 20 add_watcher() i=5 2023-09-22T13:03:11.096 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.094+0000 7f5a63560500 20 add_watcher() i=6 2023-09-22T13:03:11.096 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.096+0000 7f5a63560500 20 add_watcher() i=7 2023-09-22T13:03:11.097 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.096+0000 7f5a63560500 2 all 8 watchers are set, enabling cache 2023-09-22T13:03:11.099 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.098+0000 7f5a1ffe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:03:11.099 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.098+0000 7f5a63560500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:11.099 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.098+0000 7f5a63560500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:11.101 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.100+0000 7f5a4e241700 0 -- 172.21.15.170:0/3827483824 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x55cbdf2079d0 msgr2=0x55cbdf215260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:03:11.103 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.102+0000 7f5a63560500 5 note: GC not initialized 2023-09-22T13:03:11.104 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.103+0000 7f598effd700 20 reqs_thread_entry: start 2023-09-22T13:03:11.104 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.103+0000 7f598cff9700 20 reqs_thread_entry: start 2023-09-22T13:03:11.104 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.103+0000 7f5a63560500 20 init_complete bucket index max shards: 11 2023-09-22T13:03:11.105 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.104+0000 7f5a63560500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:03:11.105 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.104+0000 7f597effd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:03:41 2023 2023-09-22T13:03:11.105 INFO:teuthology.orchestra.run.smithi170.stderr: 2023-09-22T13:03:11.105 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.104+0000 7f597e7fc700 20 reqs_thread_entry: start 2023-09-22T13:03:11.114 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.114+0000 7f5a63560500 20 remove_watcher() i=0 2023-09-22T13:03:11.114 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.114+0000 7f5a63560500 2 removed watcher, disabling cache 2023-09-22T13:03:11.115 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.115+0000 7f5a63560500 20 remove_watcher() i=1 2023-09-22T13:03:11.116 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.116+0000 7f5a63560500 20 remove_watcher() i=2 2023-09-22T13:03:11.117 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.117+0000 7f5a63560500 20 remove_watcher() i=3 2023-09-22T13:03:11.118 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.118+0000 7f5a63560500 20 remove_watcher() i=4 2023-09-22T13:03:11.119 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.119+0000 7f5a63560500 20 remove_watcher() i=5 2023-09-22T13:03:11.120 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.120+0000 7f5a63560500 20 remove_watcher() i=6 2023-09-22T13:03:11.121 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.121+0000 7f5a63560500 20 remove_watcher() i=7 2023-09-22T13:03:11.122 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.121+0000 7f5a1ffe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:03:11.125 INFO:teuthology.orchestra.run.smithi170.stdout:[] 2023-09-22T13:03:11.126 DEBUG:tasks.util.rgw: json result: [] 2023-09-22T13:03:11.127 INFO:tasks.rgw:Configuring storage class type = LUKEWARM 2023-09-22T13:03:11.127 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM'] 2023-09-22T13:03:11.127 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM'] 2023-09-22T13:03:11.127 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zonegroup placement add --rgw-zone default --placement-id default-placement --storage-class LUKEWARM 2023-09-22T13:03:11.211 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:11.211 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:11.218 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.217+0000 7f641d457500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:11.218 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.217+0000 7f641d457500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:11.221 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.221+0000 7f641d457500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.222 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.221+0000 7f63597fa700 20 reqs_thread_entry: start 2023-09-22T13:03:11.222 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.222+0000 7f641d457500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.222 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.222+0000 7f641d457500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.223 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.222+0000 7f641d457500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.223 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.222+0000 7f641d457500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.224 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.223+0000 7f641d457500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:03:11.225 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.223+0000 7f641d457500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.225 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.224+0000 7f641d457500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:03:11.229 INFO:teuthology.orchestra.run.smithi170.stdout:[ 2023-09-22T13:03:11.229 INFO:teuthology.orchestra.run.smithi170.stdout: { 2023-09-22T13:03:11.229 INFO:teuthology.orchestra.run.smithi170.stdout: "key": "default-placement", 2023-09-22T13:03:11.229 INFO:teuthology.orchestra.run.smithi170.stdout: "val": { 2023-09-22T13:03:11.229 INFO:teuthology.orchestra.run.smithi170.stdout: "name": "default-placement", 2023-09-22T13:03:11.230 INFO:teuthology.orchestra.run.smithi170.stdout: "tags": [], 2023-09-22T13:03:11.230 INFO:teuthology.orchestra.run.smithi170.stdout: "storage_classes": [ 2023-09-22T13:03:11.230 INFO:teuthology.orchestra.run.smithi170.stdout: "LUKEWARM", 2023-09-22T13:03:11.230 INFO:teuthology.orchestra.run.smithi170.stdout: "STANDARD" 2023-09-22T13:03:11.230 INFO:teuthology.orchestra.run.smithi170.stdout: ] 2023-09-22T13:03:11.231 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:11.231 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:11.231 INFO:teuthology.orchestra.run.smithi170.stdout:] 2023-09-22T13:03:11.232 DEBUG:tasks.util.rgw: json result: [{'key': 'default-placement', 'val': {'name': 'default-placement', 'tags': [], 'storage_classes': ['LUKEWARM', 'STANDARD']}}] 2023-09-22T13:03:11.232 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM', '--data-pool', 'default.rgw.buckets.data.lukewarm'] 2023-09-22T13:03:11.232 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM', '--data-pool', 'default.rgw.buckets.data.lukewarm'] 2023-09-22T13:03:11.233 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zone placement add --rgw-zone default --placement-id default-placement --storage-class LUKEWARM --data-pool default.rgw.buckets.data.lukewarm 2023-09-22T13:03:11.281 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:11.281 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:11.288 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.287+0000 7f8edb39e500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:11.288 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.287+0000 7f8edb39e500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:11.292 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.291+0000 7f8e0bfff700 20 reqs_thread_entry: start 2023-09-22T13:03:11.292 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.291+0000 7f8edb39e500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.293 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.292+0000 7f8edb39e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.293 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.292+0000 7f8edb39e500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.293 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.293+0000 7f8edb39e500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:03:11.293 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.293+0000 7f8edb39e500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.294 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.293+0000 7f8edb39e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.294 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.293+0000 7f8edb39e500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.294 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.293+0000 7f8edb39e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.294 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.293+0000 7f8edb39e500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.295 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.294+0000 7f8edb39e500 20 rados_obj.operate() r=0 bl.length=382 2023-09-22T13:03:11.299 INFO:teuthology.orchestra.run.smithi170.stdout:{ 2023-09-22T13:03:11.299 INFO:teuthology.orchestra.run.smithi170.stdout: "id": "772ce3cc-30ad-4ea4-82fa-9614e9a1eee1", 2023-09-22T13:03:11.299 INFO:teuthology.orchestra.run.smithi170.stdout: "name": "default", 2023-09-22T13:03:11.299 INFO:teuthology.orchestra.run.smithi170.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:03:11.299 INFO:teuthology.orchestra.run.smithi170.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:03:11.299 INFO:teuthology.orchestra.run.smithi170.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:03:11.299 INFO:teuthology.orchestra.run.smithi170.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "system_key": { 2023-09-22T13:03:11.300 INFO:teuthology.orchestra.run.smithi170.stdout: "access_key": "", 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: "secret_key": "" 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: "placement_pools": [ 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: { 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: "key": "default-placement", 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: "val": { 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: "storage_classes": { 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: "LUKEWARM": { 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: "data_pool": "default.rgw.buckets.data.lukewarm" 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:03:11.301 INFO:teuthology.orchestra.run.smithi170.stdout: "STANDARD": { 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout: "compression_type": "random" 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout: "index_type": 0 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout: ], 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout: "realm_id": "", 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:03:11.302 INFO:teuthology.orchestra.run.smithi170.stdout:} 2023-09-22T13:03:11.303 DEBUG:tasks.util.rgw: json result: {'id': '772ce3cc-30ad-4ea4-82fa-9614e9a1eee1', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'LUKEWARM': {'data_pool': 'default.rgw.buckets.data.lukewarm'}, 'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:03:11.303 INFO:tasks.rgw:Configuring storage class type = FROZEN 2023-09-22T13:03:11.304 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN'] 2023-09-22T13:03:11.304 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN'] 2023-09-22T13:03:11.304 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zonegroup placement add --rgw-zone default --placement-id default-placement --storage-class FROZEN 2023-09-22T13:03:11.349 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:11.349 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:11.355 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.355+0000 7f6a1cde0500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:11.355 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.355+0000 7f6a1cde0500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:11.360 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.359+0000 7f6951ffb700 20 reqs_thread_entry: start 2023-09-22T13:03:11.360 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.359+0000 7f6a1cde0500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.360 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.360+0000 7f6a1cde0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.360 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.360+0000 7f6a1cde0500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.361 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.360+0000 7f6a1cde0500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.361 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.360+0000 7f6a1cde0500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.362 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.361+0000 7f6a1cde0500 20 rados_obj.operate() r=0 bl.length=382 2023-09-22T13:03:11.362 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.361+0000 7f6a1cde0500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.362 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.362+0000 7f6a1cde0500 20 rados_obj.operate() r=0 bl.length=956 2023-09-22T13:03:11.366 INFO:teuthology.orchestra.run.smithi170.stdout:[ 2023-09-22T13:03:11.366 INFO:teuthology.orchestra.run.smithi170.stdout: { 2023-09-22T13:03:11.366 INFO:teuthology.orchestra.run.smithi170.stdout: "key": "default-placement", 2023-09-22T13:03:11.367 INFO:teuthology.orchestra.run.smithi170.stdout: "val": { 2023-09-22T13:03:11.367 INFO:teuthology.orchestra.run.smithi170.stdout: "name": "default-placement", 2023-09-22T13:03:11.367 INFO:teuthology.orchestra.run.smithi170.stdout: "tags": [], 2023-09-22T13:03:11.367 INFO:teuthology.orchestra.run.smithi170.stdout: "storage_classes": [ 2023-09-22T13:03:11.367 INFO:teuthology.orchestra.run.smithi170.stdout: "FROZEN", 2023-09-22T13:03:11.367 INFO:teuthology.orchestra.run.smithi170.stdout: "LUKEWARM", 2023-09-22T13:03:11.367 INFO:teuthology.orchestra.run.smithi170.stdout: "STANDARD" 2023-09-22T13:03:11.367 INFO:teuthology.orchestra.run.smithi170.stdout: ] 2023-09-22T13:03:11.367 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:11.367 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:11.368 INFO:teuthology.orchestra.run.smithi170.stdout:] 2023-09-22T13:03:11.368 DEBUG:tasks.util.rgw: json result: [{'key': 'default-placement', 'val': {'name': 'default-placement', 'tags': [], 'storage_classes': ['FROZEN', 'LUKEWARM', 'STANDARD']}}] 2023-09-22T13:03:11.368 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN', '--data-pool', 'default.rgw.buckets.data.frozen'] 2023-09-22T13:03:11.369 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN', '--data-pool', 'default.rgw.buckets.data.frozen'] 2023-09-22T13:03:11.369 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zone placement add --rgw-zone default --placement-id default-placement --storage-class FROZEN --data-pool default.rgw.buckets.data.frozen 2023-09-22T13:03:11.416 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:11.416 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:11.422 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.422+0000 7fbba1e44500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:11.423 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.422+0000 7fbba1e44500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:11.426 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.426+0000 7fbad67fc700 20 reqs_thread_entry: start 2023-09-22T13:03:11.427 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.426+0000 7fbba1e44500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.427 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.427+0000 7fbba1e44500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.428 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.427+0000 7fbba1e44500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.428 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.428+0000 7fbba1e44500 20 rados_obj.operate() r=0 bl.length=956 2023-09-22T13:03:11.428 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.428+0000 7fbba1e44500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.429 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.428+0000 7fbba1e44500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.429 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.428+0000 7fbba1e44500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.429 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.429+0000 7fbba1e44500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.430 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.429+0000 7fbba1e44500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.430 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:03:11.430+0000 7fbba1e44500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:03:11.434 INFO:teuthology.orchestra.run.smithi170.stdout:{ 2023-09-22T13:03:11.434 INFO:teuthology.orchestra.run.smithi170.stdout: "id": "772ce3cc-30ad-4ea4-82fa-9614e9a1eee1", 2023-09-22T13:03:11.434 INFO:teuthology.orchestra.run.smithi170.stdout: "name": "default", 2023-09-22T13:03:11.434 INFO:teuthology.orchestra.run.smithi170.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:03:11.434 INFO:teuthology.orchestra.run.smithi170.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:03:11.435 INFO:teuthology.orchestra.run.smithi170.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: "system_key": { 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: "access_key": "", 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: "secret_key": "" 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: "placement_pools": [ 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: { 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: "key": "default-placement", 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: "val": { 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: "storage_classes": { 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: "FROZEN": { 2023-09-22T13:03:11.436 INFO:teuthology.orchestra.run.smithi170.stdout: "data_pool": "default.rgw.buckets.data.frozen" 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: "LUKEWARM": { 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: "data_pool": "default.rgw.buckets.data.lukewarm" 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: "STANDARD": { 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: "compression_type": "random" 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: "index_type": 0 2023-09-22T13:03:11.437 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:11.438 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:03:11.438 INFO:teuthology.orchestra.run.smithi170.stdout: ], 2023-09-22T13:03:11.438 INFO:teuthology.orchestra.run.smithi170.stdout: "realm_id": "", 2023-09-22T13:03:11.438 INFO:teuthology.orchestra.run.smithi170.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:03:11.438 INFO:teuthology.orchestra.run.smithi170.stdout:} 2023-09-22T13:03:11.439 DEBUG:tasks.util.rgw: json result: {'id': '772ce3cc-30ad-4ea4-82fa-9614e9a1eee1', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'FROZEN': {'data_pool': 'default.rgw.buckets.data.frozen'}, 'LUKEWARM': {'data_pool': 'default.rgw.buckets.data.lukewarm'}, 'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:03:11.439 INFO:tasks.util.rgw:rgwadmin: client.1 : ['user', 'list'] 2023-09-22T13:03:11.439 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'user', 'list'] 2023-09-22T13:03:11.439 DEBUG:teuthology.orchestra.run.smithi191:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph user list 2023-09-22T13:03:11.478 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:11.478 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:11.484 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.483+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.487 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.485+0000 7f8f5ebd3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.487 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.485+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.487 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.485+0000 7f8f5ebd3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.487 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.485+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.487 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.486+0000 7f8f5ebd3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.487 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.486+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.487 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.486+0000 7f8f5ebd3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.498 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.497+0000 7f8f5ebd3500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:03:11.498 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.497+0000 7f8f5ebd3500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:03:11.498 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.497+0000 7f8f5ebd3500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:03:11.499 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.497+0000 7f8f5ebd3500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:03:11.499 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.497+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.499 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.497+0000 7f8f5ebd3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.499 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.497+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.500 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.498+0000 7f8f5ebd3500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:03:11.500 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.498+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.500 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.498+0000 7f8f5ebd3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.500 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.498+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.500 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.498+0000 7f8f5ebd3500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:03:11.500 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.498+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.500 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.498+0000 7f8f5ebd3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.501 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.498+0000 7f8f5ebd3500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:03:11.501 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.498+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.501 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.499+0000 7f8f5ebd3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.501 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.499+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.501 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.499+0000 7f8f5ebd3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.501 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.499+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.502 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.499+0000 7f8f5ebd3500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:03:11.502 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.499+0000 7f8f5ebd3500 20 zonegroup default 2023-09-22T13:03:11.503 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.499+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.503 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.499+0000 7f8f5ebd3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.503 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.499+0000 7f8f5ebd3500 10 Cannot find current period zone using local zone 2023-09-22T13:03:11.503 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.499+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.503 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.500+0000 7f8f5ebd3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.503 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.500+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.503 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.500+0000 7f8f5ebd3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.503 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.500+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.503 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.500+0000 7f8f5ebd3500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:03:11.503 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.500+0000 7f8f5ebd3500 20 zone default found 2023-09-22T13:03:11.503 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.500+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.504 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.501+0000 7f8f5ebd3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.504 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.501+0000 7f8f5ebd3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.504 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.501+0000 7f8f5ebd3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.504 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.501+0000 7f8f5ebd3500 20 started sync module instance, tier type = 2023-09-22T13:03:11.504 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.501+0000 7f8f5ebd3500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:03:11.512 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.511+0000 7f8f5ebd3500 20 add_watcher() i=0 2023-09-22T13:03:11.512 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.511+0000 7f8f5ebd3500 20 add_watcher() i=1 2023-09-22T13:03:11.512 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.511+0000 7f8f5ebd3500 20 add_watcher() i=2 2023-09-22T13:03:11.512 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.511+0000 7f8f5ebd3500 20 add_watcher() i=3 2023-09-22T13:03:11.513 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.511+0000 7f8f5ebd3500 20 add_watcher() i=4 2023-09-22T13:03:11.513 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.511+0000 7f8f5ebd3500 20 add_watcher() i=5 2023-09-22T13:03:11.513 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.511+0000 7f8f5ebd3500 20 add_watcher() i=6 2023-09-22T13:03:11.513 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.512+0000 7f8f498b4700 0 -- 172.21.15.191:0/1415637973 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f8f40026900 msgr2=0x7f8f400243d0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:03:11.513 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.512+0000 7f8f5ebd3500 20 add_watcher() i=7 2023-09-22T13:03:11.513 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.512+0000 7f8f5ebd3500 2 all 8 watchers are set, enabling cache 2023-09-22T13:03:11.515 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.514+0000 7f8f1bfdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:03:11.515 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.514+0000 7f8f5ebd3500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:11.516 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.514+0000 7f8f5ebd3500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:11.518 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.517+0000 7f8f498b4700 0 -- 172.21.15.191:0/3720718266 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x5620090c0040 msgr2=0x5620090cd880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:03:11.520 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.519+0000 7f8f5ebd3500 5 note: GC not initialized 2023-09-22T13:03:11.521 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.520+0000 7f8e8affd700 20 reqs_thread_entry: start 2023-09-22T13:03:11.521 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.520+0000 7f8e88ff9700 20 reqs_thread_entry: start 2023-09-22T13:03:11.521 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.520+0000 7f8f5ebd3500 20 init_complete bucket index max shards: 11 2023-09-22T13:03:11.522 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.520+0000 7f8f5ebd3500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:03:11.522 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.520+0000 7f8e7effd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:03:41 2023 2023-09-22T13:03:11.522 INFO:teuthology.orchestra.run.smithi191.stderr: 2023-09-22T13:03:11.522 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.520+0000 7f8e7e7fc700 20 reqs_thread_entry: start 2023-09-22T13:03:11.531 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.530+0000 7f8f5ebd3500 20 remove_watcher() i=0 2023-09-22T13:03:11.531 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.530+0000 7f8f5ebd3500 2 removed watcher, disabling cache 2023-09-22T13:03:11.532 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.531+0000 7f8f5ebd3500 20 remove_watcher() i=1 2023-09-22T13:03:11.533 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.532+0000 7f8f5ebd3500 20 remove_watcher() i=2 2023-09-22T13:03:11.534 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.533+0000 7f8f5ebd3500 20 remove_watcher() i=3 2023-09-22T13:03:11.535 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.534+0000 7f8f5ebd3500 20 remove_watcher() i=4 2023-09-22T13:03:11.536 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.535+0000 7f8f5ebd3500 20 remove_watcher() i=5 2023-09-22T13:03:11.537 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.536+0000 7f8f5ebd3500 20 remove_watcher() i=6 2023-09-22T13:03:11.538 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.537+0000 7f8f5ebd3500 20 remove_watcher() i=7 2023-09-22T13:03:11.539 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.538+0000 7f8f1bfdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:03:11.542 INFO:teuthology.orchestra.run.smithi191.stdout:[] 2023-09-22T13:03:11.543 DEBUG:tasks.util.rgw: json result: [] 2023-09-22T13:03:11.543 INFO:tasks.rgw:Configuring storage class type = LUKEWARM 2023-09-22T13:03:11.543 INFO:tasks.util.rgw:rgwadmin: client.1 : ['zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM'] 2023-09-22T13:03:11.544 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM'] 2023-09-22T13:03:11.544 DEBUG:teuthology.orchestra.run.smithi191:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph zonegroup placement add --rgw-zone default --placement-id default-placement --storage-class LUKEWARM 2023-09-22T13:03:11.629 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:11.629 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:11.635 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.634+0000 7f156ffe3500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:11.635 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.634+0000 7f156ffe3500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:11.640 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.639+0000 7f14a3fff700 20 reqs_thread_entry: start 2023-09-22T13:03:11.640 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.639+0000 7f156ffe3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.641 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.640+0000 7f156ffe3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.641 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.640+0000 7f156ffe3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.641 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.640+0000 7f156ffe3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.642 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.640+0000 7f156ffe3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.642 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.641+0000 7f156ffe3500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:03:11.642 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.641+0000 7f156ffe3500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.643 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.642+0000 7f156ffe3500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:03:11.647 INFO:teuthology.orchestra.run.smithi191.stdout:[ 2023-09-22T13:03:11.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-09-22T13:03:11.648 INFO:teuthology.orchestra.run.smithi191.stdout: "key": "default-placement", 2023-09-22T13:03:11.648 INFO:teuthology.orchestra.run.smithi191.stdout: "val": { 2023-09-22T13:03:11.648 INFO:teuthology.orchestra.run.smithi191.stdout: "name": "default-placement", 2023-09-22T13:03:11.648 INFO:teuthology.orchestra.run.smithi191.stdout: "tags": [], 2023-09-22T13:03:11.648 INFO:teuthology.orchestra.run.smithi191.stdout: "storage_classes": [ 2023-09-22T13:03:11.648 INFO:teuthology.orchestra.run.smithi191.stdout: "FROZEN", 2023-09-22T13:03:11.648 INFO:teuthology.orchestra.run.smithi191.stdout: "LUKEWARM", 2023-09-22T13:03:11.648 INFO:teuthology.orchestra.run.smithi191.stdout: "STANDARD" 2023-09-22T13:03:11.648 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-09-22T13:03:11.648 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.648 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.649 INFO:teuthology.orchestra.run.smithi191.stdout:] 2023-09-22T13:03:11.649 DEBUG:tasks.util.rgw: json result: [{'key': 'default-placement', 'val': {'name': 'default-placement', 'tags': [], 'storage_classes': ['FROZEN', 'LUKEWARM', 'STANDARD']}}] 2023-09-22T13:03:11.649 INFO:tasks.util.rgw:rgwadmin: client.1 : ['zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM', '--data-pool', 'default.rgw.buckets.data.lukewarm'] 2023-09-22T13:03:11.650 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM', '--data-pool', 'default.rgw.buckets.data.lukewarm'] 2023-09-22T13:03:11.650 DEBUG:teuthology.orchestra.run.smithi191:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph zone placement add --rgw-zone default --placement-id default-placement --storage-class LUKEWARM --data-pool default.rgw.buckets.data.lukewarm 2023-09-22T13:03:11.696 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:11.697 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:11.702 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.701+0000 7f458fdd6500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:11.702 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.701+0000 7f458fdd6500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:11.706 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.705+0000 7f44c3fff700 20 reqs_thread_entry: start 2023-09-22T13:03:11.707 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.705+0000 7f458fdd6500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.707 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.706+0000 7f458fdd6500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.708 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.706+0000 7f458fdd6500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.708 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.707+0000 7f458fdd6500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:03:11.708 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.707+0000 7f458fdd6500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.709 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.707+0000 7f458fdd6500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.709 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.707+0000 7f458fdd6500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.709 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.708+0000 7f458fdd6500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.709 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.708+0000 7f458fdd6500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.710 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.709+0000 7f458fdd6500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:03:11.715 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-09-22T13:03:11.715 INFO:teuthology.orchestra.run.smithi191.stdout: "id": "772ce3cc-30ad-4ea4-82fa-9614e9a1eee1", 2023-09-22T13:03:11.715 INFO:teuthology.orchestra.run.smithi191.stdout: "name": "default", 2023-09-22T13:03:11.715 INFO:teuthology.orchestra.run.smithi191.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:03:11.715 INFO:teuthology.orchestra.run.smithi191.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:03:11.715 INFO:teuthology.orchestra.run.smithi191.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:03:11.715 INFO:teuthology.orchestra.run.smithi191.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:03:11.715 INFO:teuthology.orchestra.run.smithi191.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:03:11.715 INFO:teuthology.orchestra.run.smithi191.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: "system_key": { 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: "access_key": "", 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: "secret_key": "" 2023-09-22T13:03:11.716 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: "placement_pools": [ 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: "key": "default-placement", 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: "val": { 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: "storage_classes": { 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: "FROZEN": { 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: "data_pool": "default.rgw.buckets.data.frozen" 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: "LUKEWARM": { 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: "data_pool": "default.rgw.buckets.data.lukewarm" 2023-09-22T13:03:11.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-09-22T13:03:11.718 INFO:teuthology.orchestra.run.smithi191.stdout: "STANDARD": { 2023-09-22T13:03:11.718 INFO:teuthology.orchestra.run.smithi191.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:03:11.718 INFO:teuthology.orchestra.run.smithi191.stdout: "compression_type": "random" 2023-09-22T13:03:11.718 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.718 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-09-22T13:03:11.718 INFO:teuthology.orchestra.run.smithi191.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:03:11.718 INFO:teuthology.orchestra.run.smithi191.stdout: "index_type": 0 2023-09-22T13:03:11.718 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.718 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.718 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-09-22T13:03:11.718 INFO:teuthology.orchestra.run.smithi191.stdout: "realm_id": "", 2023-09-22T13:03:11.719 INFO:teuthology.orchestra.run.smithi191.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:03:11.719 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-09-22T13:03:11.719 DEBUG:tasks.util.rgw: json result: {'id': '772ce3cc-30ad-4ea4-82fa-9614e9a1eee1', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'FROZEN': {'data_pool': 'default.rgw.buckets.data.frozen'}, 'LUKEWARM': {'data_pool': 'default.rgw.buckets.data.lukewarm'}, 'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:03:11.720 INFO:tasks.rgw:Configuring storage class type = FROZEN 2023-09-22T13:03:11.720 INFO:tasks.util.rgw:rgwadmin: client.1 : ['zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN'] 2023-09-22T13:03:11.720 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN'] 2023-09-22T13:03:11.720 DEBUG:teuthology.orchestra.run.smithi191:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph zonegroup placement add --rgw-zone default --placement-id default-placement --storage-class FROZEN 2023-09-22T13:03:11.765 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:11.765 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:11.771 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.770+0000 7f8ebc57d500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:11.771 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.770+0000 7f8ebc57d500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:11.775 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.774+0000 7f8e197fa700 20 reqs_thread_entry: start 2023-09-22T13:03:11.775 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.774+0000 7f8ebc57d500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.775 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.774+0000 7f8ebc57d500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.775 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.774+0000 7f8ebc57d500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.776 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.775+0000 7f8ebc57d500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.776 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.775+0000 7f8ebc57d500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.777 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.776+0000 7f8ebc57d500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:03:11.777 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.776+0000 7f8ebc57d500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.777 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.776+0000 7f8ebc57d500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:03:11.781 INFO:teuthology.orchestra.run.smithi191.stdout:[ 2023-09-22T13:03:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-09-22T13:03:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "key": "default-placement", 2023-09-22T13:03:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "val": { 2023-09-22T13:03:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "name": "default-placement", 2023-09-22T13:03:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "tags": [], 2023-09-22T13:03:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "storage_classes": [ 2023-09-22T13:03:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "FROZEN", 2023-09-22T13:03:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "LUKEWARM", 2023-09-22T13:03:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "STANDARD" 2023-09-22T13:03:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2023-09-22T13:03:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.783 INFO:teuthology.orchestra.run.smithi191.stdout:] 2023-09-22T13:03:11.783 DEBUG:tasks.util.rgw: json result: [{'key': 'default-placement', 'val': {'name': 'default-placement', 'tags': [], 'storage_classes': ['FROZEN', 'LUKEWARM', 'STANDARD']}}] 2023-09-22T13:03:11.783 INFO:tasks.util.rgw:rgwadmin: client.1 : ['zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN', '--data-pool', 'default.rgw.buckets.data.frozen'] 2023-09-22T13:03:11.784 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'ceph', 'zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN', '--data-pool', 'default.rgw.buckets.data.frozen'] 2023-09-22T13:03:11.784 DEBUG:teuthology.orchestra.run.smithi191:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster ceph zone placement add --rgw-zone default --placement-id default-placement --storage-class FROZEN --data-pool default.rgw.buckets.data.frozen 2023-09-22T13:03:11.831 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:03:11.831 INFO:teuthology.orchestra.run.smithi191.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:03:11.837 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.836+0000 7fdb6796e500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:03:11.837 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.836+0000 7fdb6796e500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:03:11.840 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.839+0000 7fda9bfff700 20 reqs_thread_entry: start 2023-09-22T13:03:11.841 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.839+0000 7fdb6796e500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.841 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.840+0000 7fdb6796e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.841 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.840+0000 7fdb6796e500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.842 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.841+0000 7fdb6796e500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:03:11.842 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.841+0000 7fdb6796e500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.843 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.841+0000 7fdb6796e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:03:11.843 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.841+0000 7fdb6796e500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.843 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.842+0000 7fdb6796e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:03:11.843 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.842+0000 7fdb6796e500 20 rados->read ofs=0 len=0 2023-09-22T13:03:11.843 INFO:teuthology.orchestra.run.smithi191.stderr:2023-09-22T13:03:11.842+0000 7fdb6796e500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:03:11.848 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2023-09-22T13:03:11.848 INFO:teuthology.orchestra.run.smithi191.stdout: "id": "772ce3cc-30ad-4ea4-82fa-9614e9a1eee1", 2023-09-22T13:03:11.848 INFO:teuthology.orchestra.run.smithi191.stdout: "name": "default", 2023-09-22T13:03:11.848 INFO:teuthology.orchestra.run.smithi191.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:03:11.848 INFO:teuthology.orchestra.run.smithi191.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:03:11.848 INFO:teuthology.orchestra.run.smithi191.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:03:11.848 INFO:teuthology.orchestra.run.smithi191.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:03:11.848 INFO:teuthology.orchestra.run.smithi191.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:03:11.848 INFO:teuthology.orchestra.run.smithi191.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:03:11.849 INFO:teuthology.orchestra.run.smithi191.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:03:11.849 INFO:teuthology.orchestra.run.smithi191.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:03:11.849 INFO:teuthology.orchestra.run.smithi191.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:03:11.849 INFO:teuthology.orchestra.run.smithi191.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:03:11.849 INFO:teuthology.orchestra.run.smithi191.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:03:11.849 INFO:teuthology.orchestra.run.smithi191.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:03:11.849 INFO:teuthology.orchestra.run.smithi191.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:03:11.849 INFO:teuthology.orchestra.run.smithi191.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:03:11.849 INFO:teuthology.orchestra.run.smithi191.stdout: "system_key": { 2023-09-22T13:03:11.849 INFO:teuthology.orchestra.run.smithi191.stdout: "access_key": "", 2023-09-22T13:03:11.849 INFO:teuthology.orchestra.run.smithi191.stdout: "secret_key": "" 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: "placement_pools": [ 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: { 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: "key": "default-placement", 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: "val": { 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: "storage_classes": { 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: "FROZEN": { 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: "data_pool": "default.rgw.buckets.data.frozen" 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: "LUKEWARM": { 2023-09-22T13:03:11.850 INFO:teuthology.orchestra.run.smithi191.stdout: "data_pool": "default.rgw.buckets.data.lukewarm" 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: "STANDARD": { 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: "compression_type": "random" 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: "index_type": 0 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: } 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2023-09-22T13:03:11.851 INFO:teuthology.orchestra.run.smithi191.stdout: "realm_id": "", 2023-09-22T13:03:11.852 INFO:teuthology.orchestra.run.smithi191.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:03:11.852 INFO:teuthology.orchestra.run.smithi191.stdout:} 2023-09-22T13:03:11.852 DEBUG:tasks.util.rgw: json result: {'id': '772ce3cc-30ad-4ea4-82fa-9614e9a1eee1', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'FROZEN': {'data_pool': 'default.rgw.buckets.data.frozen'}, 'LUKEWARM': {'data_pool': 'default.rgw.buckets.data.lukewarm'}, 'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:03:11.853 INFO:tasks.rgw:Starting rgw... 2023-09-22T13:03:11.853 INFO:tasks.rgw:rgw client.0 config is {'ssl certificate': 'rgw.client.0'} 2023-09-22T13:03:11.853 INFO:tasks.rgw:Using civetweb as radosgw frontend 2023-09-22T13:03:11.853 INFO:tasks.rgw.client.0:Restarting daemon 2023-09-22T13:03:11.853 DEBUG:teuthology.orchestra.run.smithi170:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'civetweb ssl_certificate=/home/ubuntu/cephtest/ca/rgw.client.0.crt port=443s' -n client.0 --cluster ceph -k /etc/ceph/ceph.client.0.keyring --log-file /var/log/ceph/rgw.ceph.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.ceph.client.0.sock --foreground | sudo tee /var/log/ceph/rgw.ceph.client.0.stdout 2>&1 2023-09-22T13:03:11.855 INFO:tasks.rgw.client.0:Started 2023-09-22T13:03:11.856 INFO:tasks.rgw:rgw client.1 config is {'ssl certificate': 'rgw.client.1'} 2023-09-22T13:03:11.856 INFO:tasks.rgw:Using civetweb as radosgw frontend 2023-09-22T13:03:11.856 INFO:tasks.rgw.client.1:Restarting daemon 2023-09-22T13:03:11.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'civetweb ssl_certificate=/home/ubuntu/cephtest/ca/rgw.client.1.crt port=443s' -n client.1 --cluster ceph -k /etc/ceph/ceph.client.1.keyring --log-file /var/log/ceph/rgw.ceph.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.ceph.client.1.sock --foreground | sudo tee /var/log/ceph/rgw.ceph.client.1.stdout 2>&1 2023-09-22T13:03:11.858 INFO:tasks.rgw.client.1:Started 2023-09-22T13:03:11.858 INFO:tasks.rgw:Polling client.0 until it starts accepting connections on https://smithi170.front.sepia.ceph.com:443/ 2023-09-22T13:03:11.859 DEBUG:teuthology.orchestra.run.smithi170:> curl https://smithi170.front.sepia.ceph.com:443/ 2023-09-22T13:03:11.958 INFO:teuthology.orchestra.run.smithi170.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2023-09-22T13:03:11.958 INFO:teuthology.orchestra.run.smithi170.stderr: Dload Upload Total Spent Left Speed 2023-09-22T13:03:11.960 DEBUG:teuthology.orchestra.run:got remote process result: 7 2023-09-22T13:03:11.960 INFO:teuthology.orchestra.run.smithi170.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi170.front.sepia.ceph.com port 443: Connection refused 2023-09-22T13:03:12.065 INFO:tasks.rgw.client.1.smithi191.stdout:2023-09-22T13:03:12.064+0000 7fda385595c0 -1 IMPORTANT: the civetweb frontend is now deprecated and will be removed in a future release 2023-09-22T13:03:12.069 INFO:tasks.rgw.client.0.smithi170.stdout:2023-09-22T13:03:12.066+0000 7f15b3cc45c0 -1 IMPORTANT: the civetweb frontend is now deprecated and will be removed in a future release 2023-09-22T13:03:12.961 DEBUG:teuthology.orchestra.run.smithi170:> curl https://smithi170.front.sepia.ceph.com:443/ 2023-09-22T13:03:12.984 INFO:teuthology.orchestra.run.smithi170.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2023-09-22T13:03:12.984 INFO:teuthology.orchestra.run.smithi170.stderr: Dload Upload Total Spent Left Speed 2023-09-22T13:03:13.074 INFO:teuthology.orchestra.run.smithi170.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 2377 0 --:--:-- --:--:-- --:--:-- 2377 2023-09-22T13:03:13.075 INFO:teuthology.orchestra.run.smithi170.stdout:anonymous 2023-09-22T13:03:13.075 INFO:tasks.rgw:Polling client.1 until it starts accepting connections on https://smithi191.front.sepia.ceph.com:443/ 2023-09-22T13:03:13.075 DEBUG:teuthology.orchestra.run.smithi191:> curl https://smithi191.front.sepia.ceph.com:443/ 2023-09-22T13:03:13.114 INFO:teuthology.orchestra.run.smithi191.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2023-09-22T13:03:13.114 INFO:teuthology.orchestra.run.smithi191.stderr: Dload Upload Total Spent Left Speed 2023-09-22T13:03:13.205 INFO:teuthology.orchestra.run.smithi191.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 2351 0 --:--:-- --:--:-- --:--:-- 2351 2023-09-22T13:03:13.206 INFO:teuthology.orchestra.run.smithi191.stdout:anonymous 2023-09-22T13:03:13.206 INFO:teuthology.run_tasks:Running task workunit... 2023-09-22T13:03:13.218 INFO:tasks.workunit:Pulling workunits from ref 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:03:13.218 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-22T13:03:13.218 DEBUG:teuthology.orchestra.run.smithi170:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:03:13.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T13:03:13.237 INFO:teuthology.orchestra.run.smithi170.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-22T13:03:13.237 DEBUG:teuthology.orchestra.run.smithi170:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:03:13.295 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-22T13:03:13.295 DEBUG:teuthology.orchestra.run.smithi170:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-22T13:03:13.353 INFO:tasks.workunit:timeout=3h 2023-09-22T13:03:13.353 INFO:tasks.workunit:cleanup=True 2023-09-22T13:03:13.354 DEBUG:teuthology.orchestra.run.smithi170:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:03:13.412 INFO:tasks.workunit.client.0.smithi170.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-22T13:04:13.633 INFO:tasks.workunit.client.0.smithi170.stderr:Note: switching to '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'. 2023-09-22T13:04:13.634 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:13.634 INFO:tasks.workunit.client.0.smithi170.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2023-09-22T13:04:13.634 INFO:tasks.workunit.client.0.smithi170.stderr:changes and commit them, and you can discard any commits you make in this 2023-09-22T13:04:13.634 INFO:tasks.workunit.client.0.smithi170.stderr:state without impacting any branches by switching back to a branch. 2023-09-22T13:04:13.634 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:13.634 INFO:tasks.workunit.client.0.smithi170.stderr:If you want to create a new branch to retain commits you create, you may 2023-09-22T13:04:13.634 INFO:tasks.workunit.client.0.smithi170.stderr:do so (now or later) by using -c with the switch command. Example: 2023-09-22T13:04:13.634 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:13.634 INFO:tasks.workunit.client.0.smithi170.stderr: git switch -c 2023-09-22T13:04:13.635 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:13.635 INFO:tasks.workunit.client.0.smithi170.stderr:Or undo this operation with: 2023-09-22T13:04:13.635 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:13.635 INFO:tasks.workunit.client.0.smithi170.stderr: git switch - 2023-09-22T13:04:13.635 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:13.635 INFO:tasks.workunit.client.0.smithi170.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2023-09-22T13:04:13.635 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:13.635 INFO:tasks.workunit.client.0.smithi170.stderr:HEAD is now at 31c126b7c1c rgw: radosgw-admin bucket check should only print index entries with --check-objects flag 2023-09-22T13:04:13.640 DEBUG:teuthology.orchestra.run.smithi170:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-22T13:04:13.720 INFO:tasks.workunit.client.0.smithi170.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-22T13:04:13.722 INFO:tasks.workunit.client.0.smithi170.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:04:13.722 INFO:tasks.workunit.client.0.smithi170.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-22T13:04:14.289 INFO:tasks.workunit.client.0.smithi170.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-22T13:04:14.355 INFO:tasks.workunit.client.0.smithi170.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-22T13:04:14.393 INFO:tasks.workunit.client.0.smithi170.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:04:14.395 INFO:tasks.workunit.client.0.smithi170.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:04:14.395 INFO:tasks.workunit.client.0.smithi170.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-22T13:04:14.432 INFO:tasks.workunit.client.0.smithi170.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:04:14.435 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:04:14.435 DEBUG:teuthology.orchestra.run.smithi170:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-22T13:04:14.493 INFO:tasks.workunit:Running workunits matching rgw/run-bucket-check.sh on client.0... 2023-09-22T13:04:14.494 INFO:tasks.workunit:Running workunit rgw/run-bucket-check.sh... 2023-09-22T13:04:14.495 DEBUG:teuthology.orchestra.run.smithi170:workunit test rgw/run-bucket-check.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/run-bucket-check.sh 2023-09-22T13:04:14.556 INFO:tasks.workunit.client.0.smithi170.stderr:++ dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/run-bucket-check.sh 2023-09-22T13:04:14.556 INFO:tasks.workunit.client.0.smithi170.stderr:+ mydir=/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw 2023-09-22T13:04:14.557 INFO:tasks.workunit.client.0.smithi170.stderr:+ python3 -m venv /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw 2023-09-22T13:04:16.050 INFO:tasks.workunit.client.0.smithi170.stderr:+ source /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/bin/activate 2023-09-22T13:04:16.050 INFO:tasks.workunit.client.0.smithi170.stderr:++ deactivate nondestructive 2023-09-22T13:04:16.050 INFO:tasks.workunit.client.0.smithi170.stderr:++ '[' -n '' ']' 2023-09-22T13:04:16.051 INFO:tasks.workunit.client.0.smithi170.stderr:++ '[' -n '' ']' 2023-09-22T13:04:16.051 INFO:tasks.workunit.client.0.smithi170.stderr:++ '[' -n /usr/bin/bash -o -n '' ']' 2023-09-22T13:04:16.051 INFO:tasks.workunit.client.0.smithi170.stderr:++ hash -r 2023-09-22T13:04:16.051 INFO:tasks.workunit.client.0.smithi170.stderr:++ '[' -n '' ']' 2023-09-22T13:04:16.051 INFO:tasks.workunit.client.0.smithi170.stderr:++ unset VIRTUAL_ENV 2023-09-22T13:04:16.051 INFO:tasks.workunit.client.0.smithi170.stderr:++ '[' '!' nondestructive = nondestructive ']' 2023-09-22T13:04:16.051 INFO:tasks.workunit.client.0.smithi170.stderr:++ VIRTUAL_ENV=/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw 2023-09-22T13:04:16.051 INFO:tasks.workunit.client.0.smithi170.stderr:++ export VIRTUAL_ENV 2023-09-22T13:04:16.051 INFO:tasks.workunit.client.0.smithi170.stderr:++ _OLD_VIRTUAL_PATH=/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin 2023-09-22T13:04:16.051 INFO:tasks.workunit.client.0.smithi170.stderr:++ PATH=/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/bin:/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin 2023-09-22T13:04:16.051 INFO:tasks.workunit.client.0.smithi170.stderr:++ export PATH 2023-09-22T13:04:16.052 INFO:tasks.workunit.client.0.smithi170.stderr:++ '[' -n '' ']' 2023-09-22T13:04:16.052 INFO:tasks.workunit.client.0.smithi170.stderr:++ '[' -z '' ']' 2023-09-22T13:04:16.052 INFO:tasks.workunit.client.0.smithi170.stderr:++ _OLD_VIRTUAL_PS1= 2023-09-22T13:04:16.052 INFO:tasks.workunit.client.0.smithi170.stderr:++ '[' 'x(rgw) ' '!=' x ']' 2023-09-22T13:04:16.052 INFO:tasks.workunit.client.0.smithi170.stderr:++ PS1='(rgw) ' 2023-09-22T13:04:16.052 INFO:tasks.workunit.client.0.smithi170.stderr:++ export PS1 2023-09-22T13:04:16.052 INFO:tasks.workunit.client.0.smithi170.stderr:++ '[' -n /usr/bin/bash -o -n '' ']' 2023-09-22T13:04:16.052 INFO:tasks.workunit.client.0.smithi170.stderr:++ hash -r 2023-09-22T13:04:16.052 INFO:tasks.workunit.client.0.smithi170.stderr:+ pip install pip --upgrade 2023-09-22T13:04:16.647 INFO:tasks.workunit.client.0.smithi170.stdout:Collecting pip 2023-09-22T13:04:16.857 INFO:tasks.workunit.client.0.smithi170.stdout: Downloading https://files.pythonhosted.org/packages/a4/6d/6463d49a933f547439d6b5b98b46af8742cc03ae83543e4d7688c2420f8b/pip-21.3.1-py3-none-any.whl (1.7MB) 2023-09-22T13:04:17.328 INFO:tasks.workunit.client.0.smithi170.stdout:Installing collected packages: pip 2023-09-22T13:04:17.328 INFO:tasks.workunit.client.0.smithi170.stdout: Found existing installation: pip 9.0.3 2023-09-22T13:04:17.469 INFO:tasks.workunit.client.0.smithi170.stdout: Uninstalling pip-9.0.3: 2023-09-22T13:04:17.505 INFO:tasks.workunit.client.0.smithi170.stdout: Successfully uninstalled pip-9.0.3 2023-09-22T13:04:18.149 INFO:tasks.workunit.client.0.smithi170.stdout:Successfully installed pip-21.3.1 2023-09-22T13:04:18.355 INFO:tasks.workunit.client.0.smithi170.stderr:You are using pip version 21.3.1, however version 23.2.1 is available. 2023-09-22T13:04:18.355 INFO:tasks.workunit.client.0.smithi170.stderr:You should consider upgrading via the 'pip install --upgrade pip' command. 2023-09-22T13:04:18.387 INFO:tasks.workunit.client.0.smithi170.stderr:+ pip install boto3 2023-09-22T13:04:19.713 INFO:tasks.workunit.client.0.smithi170.stdout:Collecting boto3 2023-09-22T13:04:19.887 INFO:tasks.workunit.client.0.smithi170.stdout: Downloading boto3-1.23.10-py3-none-any.whl (132 kB) 2023-09-22T13:04:20.359 INFO:tasks.workunit.client.0.smithi170.stdout:Collecting jmespath<2.0.0,>=0.7.1 2023-09-22T13:04:20.403 INFO:tasks.workunit.client.0.smithi170.stdout: Downloading jmespath-0.10.0-py2.py3-none-any.whl (24 kB) 2023-09-22T13:04:20.494 INFO:tasks.workunit.client.0.smithi170.stdout:Collecting s3transfer<0.6.0,>=0.5.0 2023-09-22T13:04:20.516 INFO:tasks.workunit.client.0.smithi170.stdout: Downloading s3transfer-0.5.2-py3-none-any.whl (79 kB) 2023-09-22T13:04:21.575 INFO:tasks.workunit.client.0.smithi170.stdout:Collecting botocore<1.27.0,>=1.26.10 2023-09-22T13:04:21.609 INFO:tasks.workunit.client.0.smithi170.stdout: Downloading botocore-1.26.10-py3-none-any.whl (8.8 MB) 2023-09-22T13:04:22.675 INFO:tasks.workunit.client.0.smithi170.stdout:Collecting urllib3<1.27,>=1.25.4 2023-09-22T13:04:22.711 INFO:tasks.workunit.client.0.smithi170.stdout: Downloading urllib3-1.26.16-py2.py3-none-any.whl (143 kB) 2023-09-22T13:04:22.837 INFO:tasks.workunit.client.0.smithi170.stdout:Collecting python-dateutil<3.0.0,>=2.1 2023-09-22T13:04:22.863 INFO:tasks.workunit.client.0.smithi170.stdout: Downloading python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB) 2023-09-22T13:04:23.158 INFO:tasks.workunit.client.0.smithi170.stdout:Collecting six>=1.5 2023-09-22T13:04:23.177 INFO:tasks.workunit.client.0.smithi170.stdout: Downloading six-1.16.0-py2.py3-none-any.whl (11 kB) 2023-09-22T13:04:23.252 INFO:tasks.workunit.client.0.smithi170.stdout:Installing collected packages: six, urllib3, python-dateutil, jmespath, botocore, s3transfer, boto3 2023-09-22T13:04:24.359 INFO:tasks.workunit.client.0.smithi170.stdout:Successfully installed boto3-1.23.10 botocore-1.26.10 jmespath-0.10.0 python-dateutil-2.8.2 s3transfer-0.5.2 six-1.16.0 urllib3-1.26.16 2023-09-22T13:04:24.662 INFO:tasks.workunit.client.0.smithi170.stderr:+ /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/bin/python3 /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_bucket_check.py 2023-09-22T13:04:24.795 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin user create --uid check-tester --display-name "Check Testing" --access-key OJODXSLNX4LUNHQG99PA --secret 3l6ffld34qaymfomuh832j94738aie2x4p2o8h6n 2023-09-22T13:04:24.825 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:24.825 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:24.833 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.832+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.835 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.834+0000 7fe9ca5b9500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:24.835 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.834+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.836 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.835+0000 7fe9ca5b9500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:24.836 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.835+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.836 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.835+0000 7fe9ca5b9500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:24.836 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.835+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.837 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.836+0000 7fe9ca5b9500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:24.850 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.847+0000 7fe9ca5b9500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:24.850 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.847+0000 7fe9ca5b9500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:24.850 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.847+0000 7fe9ca5b9500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:24.850 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.847+0000 7fe9ca5b9500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:24.850 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.847+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.850 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.847+0000 7fe9ca5b9500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:24.851 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.847+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.851 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.848+0000 7fe9ca5b9500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:24.851 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.848+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.851 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.848+0000 7fe9ca5b9500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:24.851 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.848+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.851 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.848+0000 7fe9ca5b9500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:24.851 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.848+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.851 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.848+0000 7fe9ca5b9500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:24.852 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.848+0000 7fe9ca5b9500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:24.852 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.848+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.852 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.849+0000 7fe9ca5b9500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:24.852 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.849+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.852 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.849+0000 7fe9ca5b9500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:24.852 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.849+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.849+0000 7fe9ca5b9500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:24.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.849+0000 7fe9ca5b9500 20 zonegroup default 2023-09-22T13:04:24.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.849+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.850+0000 7fe9ca5b9500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:24.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.850+0000 7fe9ca5b9500 10 Cannot find current period zone using local zone 2023-09-22T13:04:24.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.850+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.850+0000 7fe9ca5b9500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:24.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.850+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.856 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.850+0000 7fe9ca5b9500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:24.856 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.850+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.856 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.850+0000 7fe9ca5b9500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:24.856 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.850+0000 7fe9ca5b9500 20 zone default found 2023-09-22T13:04:24.856 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.850+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.856 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.851+0000 7fe9ca5b9500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:24.856 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.851+0000 7fe9ca5b9500 20 rados->read ofs=0 len=0 2023-09-22T13:04:24.857 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.851+0000 7fe9ca5b9500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:24.857 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.851+0000 7fe9ca5b9500 20 started sync module instance, tier type = 2023-09-22T13:04:24.857 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.851+0000 7fe9ca5b9500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:24.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.863+0000 7fe9ca5b9500 20 add_watcher() i=0 2023-09-22T13:04:24.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.863+0000 7fe9ca5b9500 20 add_watcher() i=1 2023-09-22T13:04:24.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.863+0000 7fe9ca5b9500 20 add_watcher() i=2 2023-09-22T13:04:24.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.863+0000 7fe9ca5b9500 20 add_watcher() i=3 2023-09-22T13:04:24.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.863+0000 7fe9ca5b9500 20 add_watcher() i=4 2023-09-22T13:04:24.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.863+0000 7fe9ca5b9500 20 add_watcher() i=5 2023-09-22T13:04:24.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.863+0000 7fe9ca5b9500 20 add_watcher() i=6 2023-09-22T13:04:24.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.864+0000 7fe9ca5b9500 20 add_watcher() i=7 2023-09-22T13:04:24.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.864+0000 7fe9ca5b9500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:24.866 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.864+0000 7fe9b629c700 0 -- 172.21.15.170:0/4119150043 >> [v2:172.21.15.191:6816/144992,v1:172.21.15.191:6817/144992] conn(0x7fe9a804c5b0 msgr2=0x7fe9a804b660 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:24.867 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.866+0000 7fe9877de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:24.867 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.866+0000 7fe9ca5b9500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:24.867 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.866+0000 7fe9ca5b9500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:24.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.870+0000 7fe9b529a700 0 -- 172.21.15.170:0/3208792784 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x5603d085b560 msgr2=0x5603d086c850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:24.873 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.873+0000 7fe9ca5b9500 5 note: GC not initialized 2023-09-22T13:04:24.876 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.873+0000 7fe8f67fc700 20 reqs_thread_entry: start 2023-09-22T13:04:24.876 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.874+0000 7fe8e7fff700 20 reqs_thread_entry: start 2023-09-22T13:04:24.876 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.874+0000 7fe9ca5b9500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:24.876 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.874+0000 7fe9ca5b9500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:24.876 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.874+0000 7fe8e67fc700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:04:54 2023 2023-09-22T13:04:24.877 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:24.877 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.874+0000 7fe8e5ffb700 20 reqs_thread_entry: start 2023-09-22T13:04:24.877 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.874+0000 7fe9ca5b9500 20 get_system_obj_state: rctx=0x7ffdc8883198 obj=default.rgw.meta:users.uid:check-tester state=0x5603d07c0420 s->prefetch_data=0 2023-09-22T13:04:24.877 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.874+0000 7fe9ca5b9500 10 cache get: name=default.rgw.meta+users.uid+check-tester : miss 2023-09-22T13:04:24.877 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.874+0000 7fe9ca5b9500 10 cache put: name=default.rgw.meta+users.uid+check-tester info.flags=0x0 2023-09-22T13:04:24.877 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.874+0000 7fe9ca5b9500 10 adding default.rgw.meta+users.uid+check-tester to cache LRU end 2023-09-22T13:04:24.877 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.874+0000 7fe9ca5b9500 20 get_system_obj_state: rctx=0x7ffdc88831a8 obj=default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA state=0x5603d07c0420 s->prefetch_data=0 2023-09-22T13:04:24.877 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.874+0000 7fe9ca5b9500 10 cache get: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA : miss 2023-09-22T13:04:24.878 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.875+0000 7fe9ca5b9500 10 cache put: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA info.flags=0x0 2023-09-22T13:04:24.878 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.875+0000 7fe9ca5b9500 10 adding default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA to cache LRU end 2023-09-22T13:04:24.878 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.875+0000 7fe9ca5b9500 20 get_system_obj_state: rctx=0x7ffdc8882aa8 obj=default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA state=0x5603d07c0420 s->prefetch_data=0 2023-09-22T13:04:24.878 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.875+0000 7fe9ca5b9500 10 cache get: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA : hit (negative entry) 2023-09-22T13:04:24.878 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.875+0000 7fe9ca5b9500 20 get_system_obj_state: rctx=0x7ffdc8882de8 obj=default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA state=0x5603d07c0420 s->prefetch_data=0 2023-09-22T13:04:24.878 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.875+0000 7fe9ca5b9500 10 cache get: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA : hit (negative entry) 2023-09-22T13:04:24.880 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.876+0000 7fe9ca5b9500 10 cache put: name=default.rgw.meta+users.uid+check-tester info.flags=0x17 2023-09-22T13:04:24.880 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.876+0000 7fe9ca5b9500 10 moving default.rgw.meta+users.uid+check-tester to cache LRU end 2023-09-22T13:04:24.880 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.876+0000 7fe9ca5b9500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.uid:check-tester, ofs0, ns] 2023-09-22T13:04:24.880 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.876+0000 7fe9a77fe700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 163208757248 cookie 94574383072224 notifier 4577 bl.length()=559 2023-09-22T13:04:24.880 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.876+0000 7fe9a77fe700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+check-tester info.flags=0x17 2023-09-22T13:04:24.880 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.876+0000 7fe9a77fe700 10 rgw watcher librados: moving default.rgw.meta+users.uid+check-tester to cache LRU end 2023-09-22T13:04:24.881 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.878+0000 7fe9ca5b9500 10 cache put: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA info.flags=0x7 2023-09-22T13:04:24.881 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.878+0000 7fe9ca5b9500 10 moving default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA to cache LRU end 2023-09-22T13:04:24.881 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.878+0000 7fe9ca5b9500 10 distributing notification oid=default.rgw.control:notify.0 cni=[op: 0, obj: default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA, ofs0, ns] 2023-09-22T13:04:24.881 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.879+0000 7fe9a6ffd700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 163208757248 cookie 94574383064720 notifier 4577 bl.length()=186 2023-09-22T13:04:24.881 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.879+0000 7fe9a6ffd700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA info.flags=0x7 2023-09-22T13:04:24.881 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.879+0000 7fe9a6ffd700 10 rgw watcher librados: moving default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA to cache LRU end 2023-09-22T13:04:24.881 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.879+0000 7fe9a77fe700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 163208757249 cookie 94574383064720 notifier 4577 bl.length()=186 2023-09-22T13:04:24.881 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.879+0000 7fe9a77fe700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA info.flags=0x7 2023-09-22T13:04:24.882 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.879+0000 7fe9a77fe700 10 rgw watcher librados: moving default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA to cache LRU end 2023-09-22T13:04:24.882 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.881+0000 7fe9ca5b9500 20 remove_watcher() i=0 2023-09-22T13:04:24.882 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.881+0000 7fe9ca5b9500 2 removed watcher, disabling cache 2023-09-22T13:04:24.883 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.882+0000 7fe9ca5b9500 20 remove_watcher() i=1 2023-09-22T13:04:24.885 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.885+0000 7fe9ca5b9500 20 remove_watcher() i=2 2023-09-22T13:04:24.886 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.886+0000 7fe9ca5b9500 20 remove_watcher() i=3 2023-09-22T13:04:24.887 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.887+0000 7fe9ca5b9500 20 remove_watcher() i=4 2023-09-22T13:04:24.889 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.888+0000 7fe9ca5b9500 20 remove_watcher() i=5 2023-09-22T13:04:24.890 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.890+0000 7fe9ca5b9500 20 remove_watcher() i=6 2023-09-22T13:04:24.893 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.892+0000 7fe9ca5b9500 20 remove_watcher() i=7 2023-09-22T13:04:24.894 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:24.893+0000 7fe9877de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:25.255 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/ 2023-09-22T13:04:25.255 INFO:tasks.workunit.client.0.smithi170.stderr: warnings.warn(warning, PythonDeprecationWarning) 2023-09-22T13:04:25.256 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.256 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.256 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.256 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.256 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.256 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.256 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.257 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.257 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.257 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.257 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.257 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.257 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.257 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.257 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.257 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.257 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.258 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.258 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.258 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.258 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.258 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.258 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.258 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.258 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.258 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.259 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.259 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.259 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.259 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.259 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:25.259 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:25.259 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: bucket check unlinked does not report normal entries 2023-09-22T13:04:25.259 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:25.260 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --min-age-hours 0 --dump-keys 2023-09-22T13:04:25.286 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:25.286 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:25.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.294+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.299 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.299+0000 7f1aac91b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:25.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.299+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.300+0000 7f1aac91b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:25.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.300+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.300+0000 7f1aac91b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:25.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.300+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.301+0000 7f1aac91b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:25.313 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.312+0000 7f1aac91b500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:25.313 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.312+0000 7f1aac91b500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:25.314 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.312+0000 7f1aac91b500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:25.314 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.312+0000 7f1aac91b500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:25.314 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.312+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.314 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.312+0000 7f1aac91b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:25.314 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.312+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.314 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.312+0000 7f1aac91b500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:25.314 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.312+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.314 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.313+0000 7f1aac91b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:25.314 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.313+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.316 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.313+0000 7f1aac91b500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:25.317 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.313+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.317 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.313+0000 7f1aac91b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:25.317 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.313+0000 7f1aac91b500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:25.317 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.313+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.317 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.313+0000 7f1aac91b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:25.317 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.313+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.317 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.314+0000 7f1aac91b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:25.317 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.314+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.317 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.314+0000 7f1aac91b500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:25.317 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.314+0000 7f1aac91b500 20 zonegroup default 2023-09-22T13:04:25.318 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.314+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.318 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.315+0000 7f1aac91b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:25.318 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.315+0000 7f1aac91b500 10 Cannot find current period zone using local zone 2023-09-22T13:04:25.318 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.315+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.318 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.315+0000 7f1aac91b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:25.318 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.315+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.318 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.315+0000 7f1aac91b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:25.318 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.315+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.318 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.316+0000 7f1aac91b500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:25.318 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.316+0000 7f1aac91b500 20 zone default found 2023-09-22T13:04:25.318 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.316+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.319 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.316+0000 7f1aac91b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:25.319 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.316+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.319 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.316+0000 7f1aac91b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:25.319 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.316+0000 7f1aac91b500 20 started sync module instance, tier type = 2023-09-22T13:04:25.319 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.316+0000 7f1aac91b500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:25.336 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.335+0000 7f1a975fc700 0 -- 172.21.15.170:0/2821367209 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f1a88013940 msgr2=0x7f1a8802a3c0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:25.336 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.335+0000 7f1aac91b500 20 add_watcher() i=0 2023-09-22T13:04:25.337 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.335+0000 7f1aac91b500 20 add_watcher() i=1 2023-09-22T13:04:25.337 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.335+0000 7f1aac91b500 20 add_watcher() i=2 2023-09-22T13:04:25.337 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.335+0000 7f1aac91b500 20 add_watcher() i=3 2023-09-22T13:04:25.337 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.335+0000 7f1aac91b500 20 add_watcher() i=4 2023-09-22T13:04:25.337 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.335+0000 7f1aac91b500 20 add_watcher() i=5 2023-09-22T13:04:25.338 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.337+0000 7f1aac91b500 20 add_watcher() i=6 2023-09-22T13:04:25.338 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.337+0000 7f1aac91b500 20 add_watcher() i=7 2023-09-22T13:04:25.338 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.337+0000 7f1aac91b500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:25.340 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.339+0000 7f1a697e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:25.340 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.339+0000 7f1aac91b500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:25.340 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.339+0000 7f1aac91b500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:25.344 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.343+0000 7f1a97dfd700 0 -- 172.21.15.170:0/3839453779 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x5628108a1810 msgr2=0x5628108961a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:25.346 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.346+0000 7f1aac91b500 5 note: GC not initialized 2023-09-22T13:04:25.348 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f19d7fff700 20 reqs_thread_entry: start 2023-09-22T13:04:25.348 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f19d5ffb700 20 reqs_thread_entry: start 2023-09-22T13:04:25.348 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f1aac91b500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:25.348 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f1aac91b500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:25.349 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f19c3fff700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:04:55 2023 2023-09-22T13:04:25.349 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:25.349 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f19c37fe700 20 reqs_thread_entry: start 2023-09-22T13:04:25.349 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f1aac91b500 20 get_system_obj_state: rctx=0x7ffc555646d8 obj=default.rgw.meta:root:check-bucket state=0x5628108a26d0 s->prefetch_data=0 2023-09-22T13:04:25.349 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f1aac91b500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:04:25.349 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f1aac91b500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:04:25.349 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f1aac91b500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:25.349 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f1aac91b500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:25.349 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f1aac91b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:25.350 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f1aac91b500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:04:25.350 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.347+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.352 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:25.352 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:04:25.352 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:25.352 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 20 get_system_obj_state: rctx=0x7ffc555647a8 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5628108a26d0 s->prefetch_data=0 2023-09-22T13:04:25.352 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : miss 2023-09-22T13:04:25.352 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x16 2023-09-22T13:04:25.353 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:25.353 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:25.353 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:25.353 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:25.353 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:25.353 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:25.353 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:04:25.353 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.348+0000 7f1aac91b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:25.353 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.349+0000 7f1aac91b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:25.354 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.349+0000 7f1aac91b500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x13 2023-09-22T13:04:25.354 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.349+0000 7f1aac91b500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:25.354 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.349+0000 7f1aac91b500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:25.354 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.349+0000 7f1aac91b500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:25.354 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.349+0000 7f1aac91b500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 2023-09-22T13:04:25.354 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.349+0000 7f1aac91b500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:25.354 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.349+0000 7f1aac91b500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:04:25.354 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.349+0000 7f1aac91b500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:25.354 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.350+0000 7f1aac91b500 1 NOTICE: finished shard 1 (0 entries found) 2023-09-22T13:04:25.355 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.350+0000 7f1aac91b500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:25.355 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.350+0000 7f1aac91b500 1 NOTICE: finished shard 2 (0 entries found) 2023-09-22T13:04:25.355 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.350+0000 7f1aac91b500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:25.355 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.351+0000 7f1aac91b500 1 NOTICE: finished shard 3 (0 entries found) 2023-09-22T13:04:25.355 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.351+0000 7f1aac91b500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:25.355 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.351+0000 7f1aac91b500 1 NOTICE: finished shard 4 (0 entries found) 2023-09-22T13:04:25.355 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.351+0000 7f1aac91b500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:25.355 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.351+0000 7f1aac91b500 1 NOTICE: finished shard 5 (0 entries found) 2023-09-22T13:04:25.355 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.351+0000 7f1aac91b500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:25.357 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.352+0000 7f1aac91b500 1 NOTICE: finished shard 6 (0 entries found) 2023-09-22T13:04:25.357 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.352+0000 7f1aac91b500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:25.357 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.352+0000 7f1aac91b500 1 NOTICE: finished shard 7 (0 entries found) 2023-09-22T13:04:25.357 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.352+0000 7f1aac91b500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:25.357 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.353+0000 7f1aac91b500 1 NOTICE: finished shard 8 (0 entries found) 2023-09-22T13:04:25.357 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.353+0000 7f1aac91b500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:25.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.353+0000 7f1aac91b500 1 NOTICE: finished shard 9 (0 entries found) 2023-09-22T13:04:25.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.353+0000 7f1aac91b500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:25.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.354+0000 7f1aac91b500 1 NOTICE: finished shard 10 (0 entries found) 2023-09-22T13:04:25.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.354+0000 7f1aac91b500 1 NOTICE: finished all shards (0 entries found) 2023-09-22T13:04:25.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.355+0000 7f1aac91b500 20 remove_watcher() i=0 2023-09-22T13:04:25.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.355+0000 7f1aac91b500 2 removed watcher, disabling cache 2023-09-22T13:04:25.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.356+0000 7f1aac91b500 20 remove_watcher() i=1 2023-09-22T13:04:25.359 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.358+0000 7f1aac91b500 20 remove_watcher() i=2 2023-09-22T13:04:25.359 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.359+0000 7f1aac91b500 20 remove_watcher() i=3 2023-09-22T13:04:25.360 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.360+0000 7f1aac91b500 20 remove_watcher() i=4 2023-09-22T13:04:25.361 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.361+0000 7f1aac91b500 20 remove_watcher() i=5 2023-09-22T13:04:25.362 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.362+0000 7f1aac91b500 20 remove_watcher() i=6 2023-09-22T13:04:25.363 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.363+0000 7f1aac91b500 20 remove_watcher() i=7 2023-09-22T13:04:25.364 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:25.363+0000 7f1a697e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:25.370 INFO:tasks.workunit.client.0.smithi170.stderr:ceph config set client rgw_debug_inject_set_olh_err 2 2023-09-22T13:04:25.767 INFO:tasks.workunit.client.0.smithi170.stderr:ceph config set client rgw_debug_inject_olh_cancel_modification_err true 2023-09-22T13:04:27.316 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:27.316 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:27.317 INFO:tasks.workunit.client.0.smithi170.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:04:27.318 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:04:27.343 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:27.344 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:27.352 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.351+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.354 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.354+0000 7fd52bfca500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.355 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.354+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.355 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.354+0000 7fd52bfca500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.356 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.354+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.356 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.355+0000 7fd52bfca500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.356 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.355+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.356 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.355+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.368 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.367+0000 7fd52bfca500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:27.369 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.367+0000 7fd52bfca500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:27.369 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.367+0000 7fd52bfca500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:27.369 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.367+0000 7fd52bfca500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:27.369 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.367+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.369 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.367+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.369 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.367+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.370 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.368+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:27.370 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.368+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.370 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.368+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.370 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.368+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.374 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.368+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:27.374 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.368+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.374 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.368+0000 7fd52bfca500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.374 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.368+0000 7fd52bfca500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:27.374 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.368+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.374 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.369+0000 7fd52bfca500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.375 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.369+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.375 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.369+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.375 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.369+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.375 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.369+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:27.375 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.369+0000 7fd52bfca500 20 zonegroup default 2023-09-22T13:04:27.375 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.369+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.375 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.370+0000 7fd52bfca500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.375 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.370+0000 7fd52bfca500 10 Cannot find current period zone using local zone 2023-09-22T13:04:27.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.370+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.370+0000 7fd52bfca500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.370+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.370+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.370+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.370+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:27.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.370+0000 7fd52bfca500 20 zone default found 2023-09-22T13:04:27.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.370+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.371+0000 7fd52bfca500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.377 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.371+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.377 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.371+0000 7fd52bfca500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.377 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.371+0000 7fd52bfca500 20 started sync module instance, tier type = 2023-09-22T13:04:27.377 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.371+0000 7fd52bfca500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:27.384 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.383+0000 7fd50ecab700 0 -- 172.21.15.170:0/2547711070 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7fd508023040 msgr2=0x7fd508024940 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:27.384 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.383+0000 7fd52bfca500 20 add_watcher() i=0 2023-09-22T13:04:27.384 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.383+0000 7fd52bfca500 20 add_watcher() i=1 2023-09-22T13:04:27.384 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.383+0000 7fd52bfca500 20 add_watcher() i=2 2023-09-22T13:04:27.385 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.383+0000 7fd52bfca500 20 add_watcher() i=3 2023-09-22T13:04:27.385 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.383+0000 7fd52bfca500 20 add_watcher() i=4 2023-09-22T13:04:27.385 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.383+0000 7fd52bfca500 20 add_watcher() i=5 2023-09-22T13:04:27.385 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.383+0000 7fd52bfca500 20 add_watcher() i=6 2023-09-22T13:04:27.386 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.385+0000 7fd52bfca500 20 add_watcher() i=7 2023-09-22T13:04:27.386 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.385+0000 7fd52bfca500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:27.388 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.387+0000 7fd4e8fe1700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:27.388 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.387+0000 7fd52bfca500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:27.388 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.387+0000 7fd52bfca500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:27.391 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.391+0000 7fd50ecab700 0 -- 172.21.15.170:0/3376946638 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x55db9266cee0 msgr2=0x55db92657880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:27.394 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.393+0000 7fd52bfca500 5 note: GC not initialized 2023-09-22T13:04:27.395 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.394+0000 7fd4577fe700 20 reqs_thread_entry: start 2023-09-22T13:04:27.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.394+0000 7fd4557fa700 20 reqs_thread_entry: start 2023-09-22T13:04:27.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.394+0000 7fd52bfca500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:27.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.394+0000 7fd52bfca500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:27.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.394+0000 7fd4437fe700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:04:57 2023 2023-09-22T13:04:27.396 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:27.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.394+0000 7fd442ffd700 20 reqs_thread_entry: start 2023-09-22T13:04:27.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.394+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3480 obj=default.rgw.meta:root:check-bucket state=0x55db9265d290 s->prefetch_data=0 2023-09-22T13:04:27.397 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.395+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.397 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.395+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.400 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.395+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:27.401 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.395+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3480 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.401 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.396+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.401 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.396+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.401 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.396+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.401 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.396+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.401 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.396+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.396+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.396+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3250 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.396+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.396+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.396+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.396+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.397+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.397+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.403 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.397+0000 7fd52bfca500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:27.403 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.397+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3250 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.403 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.397+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.403 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.397+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.403 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.397+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.403 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.397+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.403 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.398+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.404 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.398+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.404 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.398+0000 7fd52bfca500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:27.404 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.398+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3250 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.404 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.399+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.404 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.399+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.404 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.399+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.404 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.399+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.404 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.399+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.405 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.399+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.405 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.399+0000 7fd52bfca500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:27.405 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.399+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3250 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.405 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.400+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.405 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.400+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.405 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.400+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.405 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.400+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.405 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.400+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.406 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.400+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.406 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.400+0000 7fd52bfca500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:27.410 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.400+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3250 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.411 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.401+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.411 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.401+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.411 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.401+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.411 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.401+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.411 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.401+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.411 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.401+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.411 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.401+0000 7fd52bfca500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:27.411 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.402+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3250 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.412 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.402+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.412 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.402+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.412 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.402+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.412 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.402+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.412 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.402+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.412 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.402+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.412 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.402+0000 7fd52bfca500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:27.412 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.403+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3250 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.413 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.403+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.413 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.403+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.413 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.403+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.413 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.403+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.413 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.403+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.413 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.403+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.413 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.403+0000 7fd52bfca500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:27.413 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.404+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3250 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.414 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.404+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.414 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.404+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.414 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.404+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.414 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.404+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.414 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.404+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.414 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.404+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.414 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.404+0000 7fd52bfca500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:27.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.405+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3250 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.405+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.405+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.405+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.405+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.405+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.406+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.406+0000 7fd52bfca500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:27.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.406+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3250 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.416 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.406+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.416 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.406+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.416 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.406+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.416 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.406+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.416 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.407+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.416 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.407+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.416 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.407+0000 7fd52bfca500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:27.416 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.407+0000 7fd52bfca500 20 get_system_obj_state: rctx=0x7ffc815d3250 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55db926590c0 s->prefetch_data=0 2023-09-22T13:04:27.417 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.407+0000 7fd52bfca500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.417 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.408+0000 7fd52bfca500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.417 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.408+0000 7fd52bfca500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.417 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.408+0000 7fd52bfca500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.417 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.408+0000 7fd52bfca500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.417 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.408+0000 7fd52bfca500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.417 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.408+0000 7fd52bfca500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:27.417 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.410+0000 7fd52bfca500 20 remove_watcher() i=0 2023-09-22T13:04:27.417 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.410+0000 7fd52bfca500 2 removed watcher, disabling cache 2023-09-22T13:04:27.419 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.411+0000 7fd52bfca500 20 remove_watcher() i=1 2023-09-22T13:04:27.419 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.412+0000 7fd52bfca500 20 remove_watcher() i=2 2023-09-22T13:04:27.419 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.413+0000 7fd52bfca500 20 remove_watcher() i=3 2023-09-22T13:04:27.419 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.414+0000 7fd52bfca500 20 remove_watcher() i=4 2023-09-22T13:04:27.419 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.416+0000 7fd52bfca500 20 remove_watcher() i=5 2023-09-22T13:04:27.419 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.417+0000 7fd52bfca500 20 remove_watcher() i=6 2023-09-22T13:04:27.420 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.418+0000 7fd52bfca500 20 remove_watcher() i=7 2023-09-22T13:04:27.420 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.419+0000 7fd4e8fe1700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:27.478 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:27.478 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:27.478 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:27.478 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:27.478 INFO:tasks.workunit.client.0.smithi170.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:04:27.479 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:04:27.504 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:27.505 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:27.512 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.511+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.514 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.513+0000 7f76dc715500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.514 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.513+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.516 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.514+0000 7f76dc715500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.516 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.514+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.516 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.514+0000 7f76dc715500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.516 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.514+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.516 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.515+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.527 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.525+0000 7f76dc715500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:27.527 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.525+0000 7f76dc715500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:27.527 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.525+0000 7f76dc715500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:27.527 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.525+0000 7f76dc715500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:27.527 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.525+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.525+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.525+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.526+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:27.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.526+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.526+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.526+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.526+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:27.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.526+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.531 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.527+0000 7f76dc715500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.531 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.527+0000 7f76dc715500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:27.531 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.527+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.531 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.527+0000 7f76dc715500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.531 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.527+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.532 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.527+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.532 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.527+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.532 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.527+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:27.532 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.527+0000 7f76dc715500 20 zonegroup default 2023-09-22T13:04:27.532 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.527+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.532 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.532 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 10 Cannot find current period zone using local zone 2023-09-22T13:04:27.532 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.533 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.533 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.533 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.533 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.533 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:27.533 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 20 zone default found 2023-09-22T13:04:27.533 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.533 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.534 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.528+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.534 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.529+0000 7f76dc715500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.534 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.529+0000 7f76dc715500 20 started sync module instance, tier type = 2023-09-22T13:04:27.534 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.529+0000 7f76dc715500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:27.544 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.543+0000 7f76dc715500 20 add_watcher() i=0 2023-09-22T13:04:27.544 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.543+0000 7f76dc715500 20 add_watcher() i=1 2023-09-22T13:04:27.545 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.543+0000 7f76dc715500 20 add_watcher() i=2 2023-09-22T13:04:27.545 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.543+0000 7f76dc715500 20 add_watcher() i=3 2023-09-22T13:04:27.545 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.543+0000 7f76dc715500 20 add_watcher() i=4 2023-09-22T13:04:27.545 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.543+0000 7f76dc715500 20 add_watcher() i=5 2023-09-22T13:04:27.545 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.543+0000 7f76dc715500 20 add_watcher() i=6 2023-09-22T13:04:27.547 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.546+0000 7f76dc715500 20 add_watcher() i=7 2023-09-22T13:04:27.547 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.546+0000 7f76dc715500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:27.547 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.547+0000 7f76c83f8700 0 -- 172.21.15.170:0/3159087423 >> [v2:172.21.15.191:6816/144992,v1:172.21.15.191:6817/144992] conn(0x7f76b0034910 msgr2=0x7f76b0021410 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:27.549 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.548+0000 7f76997e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:27.549 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.549+0000 7f76dc715500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:27.550 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.549+0000 7f76dc715500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:27.553 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.552+0000 7f76c83f8700 0 -- 172.21.15.170:0/2019854825 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x55dd27cbd550 msgr2=0x55dd27c9b340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:27.754 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.754+0000 7f76c83f8700 0 --2- 172.21.15.170:0/2019854825 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x55dd27cbd550 0x55dd27c9b340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:04:27.757 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.756+0000 7f76dc715500 5 note: GC not initialized 2023-09-22T13:04:27.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.757+0000 7f7607fff700 20 reqs_thread_entry: start 2023-09-22T13:04:27.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.757+0000 7f7605ffb700 20 reqs_thread_entry: start 2023-09-22T13:04:27.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.757+0000 7f76dc715500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:27.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.757+0000 7f76dc715500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:27.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.757+0000 7f75f3fff700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:04:58 2023 2023-09-22T13:04:27.760 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:27.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.757+0000 7f75f37fe700 20 reqs_thread_entry: start 2023-09-22T13:04:27.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.757+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac880000 obj=default.rgw.meta:root:check-bucket state=0x55dd27bb7b60 s->prefetch_data=0 2023-09-22T13:04:27.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.758+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.758+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.758+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:27.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.758+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac880000 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.764 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.759+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.764 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.759+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.764 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.759+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.764 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.759+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.765 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.759+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.765 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.759+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.765 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.759+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac87fdd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.765 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.759+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.765 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.759+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.765 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.759+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.765 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.759+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.765 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.760+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.760+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.760+0000 7f76dc715500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:27.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.760+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac87fdd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.760+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.760+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.760+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.760+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.761+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.761+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.767 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.761+0000 7f76dc715500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:27.767 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.761+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac87fdd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.767 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.762+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.767 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.762+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.767 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.762+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.767 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.762+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.767 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.762+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.767 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.762+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.762+0000 7f76dc715500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:27.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.763+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac87fdd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.763+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.763+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.763+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.763+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.763+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.763+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.763+0000 7f76dc715500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:27.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.764+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac87fdd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.773 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.764+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.773 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.764+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.773 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.764+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.774 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.764+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.774 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.764+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.774 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.764+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.774 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.764+0000 7f76dc715500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:27.774 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.765+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac87fdd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.774 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.765+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.774 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.765+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.774 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.765+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.775 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.765+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.775 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.765+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.775 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.765+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.775 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.765+0000 7f76dc715500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:27.775 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.766+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac87fdd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.775 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.766+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.775 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.766+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.775 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.766+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.776 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.766+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.776 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.766+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.776 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.766+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.776 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.766+0000 7f76dc715500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:27.776 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.767+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac87fdd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.776 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.767+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.776 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.767+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.776 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.767+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.776 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.767+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.777 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.767+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.777 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.767+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.777 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.767+0000 7f76dc715500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:27.777 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.768+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac87fdd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.777 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.768+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.777 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.768+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.777 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.768+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.777 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.768+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.778 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.769+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.778 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.769+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.778 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.769+0000 7f76dc715500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:27.778 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.769+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac87fdd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.778 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.769+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.778 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.769+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.778 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.769+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.778 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.769+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.778 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.770+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.779 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.770+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.779 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.770+0000 7f76dc715500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:27.779 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.770+0000 7f76dc715500 20 get_system_obj_state: rctx=0x7fffac87fdd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55dd27bb5b20 s->prefetch_data=0 2023-09-22T13:04:27.779 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.771+0000 7f76dc715500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.779 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.771+0000 7f76dc715500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.779 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.771+0000 7f76dc715500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.779 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.771+0000 7f76dc715500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.779 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.771+0000 7f76dc715500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.779 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.771+0000 7f76dc715500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.780 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.771+0000 7f76dc715500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:27.780 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.773+0000 7f76dc715500 20 remove_watcher() i=0 2023-09-22T13:04:27.780 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.773+0000 7f76dc715500 2 removed watcher, disabling cache 2023-09-22T13:04:27.781 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.774+0000 7f76dc715500 20 remove_watcher() i=1 2023-09-22T13:04:27.781 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.775+0000 7f76dc715500 20 remove_watcher() i=2 2023-09-22T13:04:27.782 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.776+0000 7f76dc715500 20 remove_watcher() i=3 2023-09-22T13:04:27.782 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.777+0000 7f76dc715500 20 remove_watcher() i=4 2023-09-22T13:04:27.782 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.778+0000 7f76dc715500 20 remove_watcher() i=5 2023-09-22T13:04:27.782 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.779+0000 7f76dc715500 20 remove_watcher() i=6 2023-09-22T13:04:27.782 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.782+0000 7f76dc715500 20 remove_watcher() i=7 2023-09-22T13:04:27.783 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.783+0000 7f76997e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:27.844 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:27.844 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:27.844 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:27.844 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:27.844 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:27.844 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:27.844 INFO:tasks.workunit.client.0.smithi170.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:04:27.845 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object e 2023-09-22T13:04:27.869 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:27.870 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:27.877 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.877+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.879 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.879+0000 7f036c8ce500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.879 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.879+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.880 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.879+0000 7f036c8ce500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.880 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.879+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.880 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.880+0000 7f036c8ce500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.880 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.880+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.881 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.880+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.891 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.890+0000 7f036c8ce500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:27.891 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.890+0000 7f036c8ce500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:27.891 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.890+0000 7f036c8ce500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:27.891 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.890+0000 7f036c8ce500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:27.892 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.890+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.892 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.890+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.892 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.890+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.892 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.891+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:27.892 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.891+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.893 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.891+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.893 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.891+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.893 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.892+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:27.893 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.892+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.893 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.892+0000 7f036c8ce500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.893 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.892+0000 7f036c8ce500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:27.894 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.892+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.894 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.892+0000 7f036c8ce500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.894 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.892+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.894 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.892+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.894 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.892+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.895 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.893+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:27.895 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.893+0000 7f036c8ce500 20 zonegroup default 2023-09-22T13:04:27.896 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.893+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.896 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.893+0000 7f036c8ce500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.896 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.893+0000 7f036c8ce500 10 Cannot find current period zone using local zone 2023-09-22T13:04:27.896 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.893+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.896 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.893+0000 7f036c8ce500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.896 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.893+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.896 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.893+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:27.896 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.893+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.896 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.894+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:27.896 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.894+0000 7f036c8ce500 20 zone default found 2023-09-22T13:04:27.897 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.894+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.897 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.894+0000 7f036c8ce500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.897 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.894+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.897 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.894+0000 7f036c8ce500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:27.897 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.894+0000 7f036c8ce500 20 started sync module instance, tier type = 2023-09-22T13:04:27.897 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.894+0000 7f036c8ce500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:27.908 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.907+0000 7f036c8ce500 20 add_watcher() i=0 2023-09-22T13:04:27.908 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.907+0000 7f036c8ce500 20 add_watcher() i=1 2023-09-22T13:04:27.908 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.907+0000 7f036c8ce500 20 add_watcher() i=2 2023-09-22T13:04:27.908 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.907+0000 7f036c8ce500 20 add_watcher() i=3 2023-09-22T13:04:27.908 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.907+0000 7f036c8ce500 20 add_watcher() i=4 2023-09-22T13:04:27.908 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.907+0000 7f036c8ce500 20 add_watcher() i=5 2023-09-22T13:04:27.909 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.907+0000 7f036c8ce500 20 add_watcher() i=6 2023-09-22T13:04:27.909 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.908+0000 7f03575af700 0 -- 172.21.15.170:0/2056936998 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f034c01f030 msgr2=0x7f034c0306f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:27.909 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.908+0000 7f036c8ce500 20 add_watcher() i=7 2023-09-22T13:04:27.909 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.908+0000 7f036c8ce500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:27.911 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.910+0000 7f03297e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:27.911 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.910+0000 7f036c8ce500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:27.911 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.910+0000 7f036c8ce500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:27.914 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.914+0000 7f03575af700 0 -- 172.21.15.170:0/2092922056 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x55c2f91cb600 msgr2=0x55c2f91c5ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:27.917 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.916+0000 7f036c8ce500 5 note: GC not initialized 2023-09-22T13:04:27.918 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.917+0000 7f0297fff700 20 reqs_thread_entry: start 2023-09-22T13:04:27.918 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.917+0000 7f0295ffb700 20 reqs_thread_entry: start 2023-09-22T13:04:27.918 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.917+0000 7f036c8ce500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:27.918 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.917+0000 7f036c8ce500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:27.918 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.917+0000 7f0283fff700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:04:58 2023 2023-09-22T13:04:27.918 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:27.919 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.917+0000 7f02837fe700 20 reqs_thread_entry: start 2023-09-22T13:04:27.919 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.917+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0450 obj=default.rgw.meta:root:check-bucket state=0x55c2f914f430 s->prefetch_data=0 2023-09-22T13:04:27.920 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.918+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.920 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.918+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.920 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.918+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:27.920 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.918+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0450 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.920 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.918+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.920 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.918+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.918+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.918+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.919+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.919+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.919+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0220 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.919+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.919+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.919+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.919+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.919+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.919+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.922 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.919+0000 7f036c8ce500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:27.923 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.920+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0220 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.923 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.920+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.923 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.920+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.923 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.920+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.923 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.920+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.923 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.920+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.923 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.920+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.924 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.920+0000 7f036c8ce500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:27.924 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.921+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0220 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.924 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.921+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.924 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.921+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.924 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.921+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.924 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.921+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.924 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.922+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.924 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.922+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.924 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.922+0000 7f036c8ce500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:27.924 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.922+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0220 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.924 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.922+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.925 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.922+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.925 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.922+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.925 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.922+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.926 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.923+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.926 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.923+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.926 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.923+0000 7f036c8ce500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:27.926 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.923+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0220 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.926 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.924+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.926 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.924+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.927 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.924+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.927 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.924+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.927 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.924+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.927 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.924+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.927 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.924+0000 7f036c8ce500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:27.927 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.924+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0220 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.927 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.925+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.927 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.925+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.927 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.925+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.927 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.925+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.927 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.925+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.928 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.925+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.928 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.925+0000 7f036c8ce500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:27.928 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.926+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0220 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.929 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.926+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.929 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.926+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.929 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.926+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.929 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.926+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.926+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.926+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.926+0000 7f036c8ce500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.927+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0220 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.927+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.927+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.927+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.927+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.928+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.928+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.928+0000 7f036c8ce500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:27.930 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.928+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0220 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.931 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.928+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.931 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.928+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.931 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.928+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.931 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.928+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.929+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.929+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.929+0000 7f036c8ce500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:27.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.929+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0220 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.930+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.930+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.930+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.930+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.930+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.930+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.930+0000 7f036c8ce500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.930+0000 7f036c8ce500 20 get_system_obj_state: rctx=0x7fff826f0220 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55c2f90c5b60 s->prefetch_data=0 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.931+0000 7f036c8ce500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.931+0000 7f036c8ce500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.931+0000 7f036c8ce500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.931+0000 7f036c8ce500 20 rados->read ofs=0 len=0 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.931+0000 7f036c8ce500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.931+0000 7f036c8ce500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:27.933 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.931+0000 7f036c8ce500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:27.934 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.933+0000 7f036c8ce500 20 remove_watcher() i=0 2023-09-22T13:04:27.934 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.933+0000 7f036c8ce500 2 removed watcher, disabling cache 2023-09-22T13:04:27.935 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.934+0000 7f036c8ce500 20 remove_watcher() i=1 2023-09-22T13:04:27.937 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.935+0000 7f036c8ce500 20 remove_watcher() i=2 2023-09-22T13:04:27.937 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.936+0000 7f036c8ce500 20 remove_watcher() i=3 2023-09-22T13:04:27.938 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.937+0000 7f036c8ce500 20 remove_watcher() i=4 2023-09-22T13:04:27.939 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.938+0000 7f036c8ce500 20 remove_watcher() i=5 2023-09-22T13:04:27.940 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.940+0000 7f036c8ce500 20 remove_watcher() i=6 2023-09-22T13:04:27.943 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.942+0000 7f036c8ce500 20 remove_watcher() i=7 2023-09-22T13:04:27.943 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:27.943+0000 7f03297e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:28.002 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:28.002 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:28.002 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:28.002 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:28.002 INFO:tasks.workunit.client.0.smithi170.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:04:28.003 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object f 2023-09-22T13:04:28.027 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:28.027 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:28.035 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.034+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.037 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.036+0000 7f8ef406b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:28.037 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.036+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.037 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.037+0000 7f8ef406b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:28.037 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.037+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.037 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.037+0000 7f8ef406b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:28.038 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.037+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.038 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.038+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:28.048 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.047+0000 7f8ef406b500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:28.048 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.047+0000 7f8ef406b500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:28.048 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.047+0000 7f8ef406b500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:28.048 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.047+0000 7f8ef406b500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:28.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.047+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.047+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:28.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.047+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.048+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:28.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.048+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.050 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.048+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:28.050 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.048+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.050 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.049+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:28.050 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.049+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.050 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.049+0000 7f8ef406b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:28.050 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.049+0000 7f8ef406b500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:28.051 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.049+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.051 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.049+0000 7f8ef406b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:28.051 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.049+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.051 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.049+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:28.051 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.049+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.053 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.050+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:28.053 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.050+0000 7f8ef406b500 20 zonegroup default 2023-09-22T13:04:28.053 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.050+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.053 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.050+0000 7f8ef406b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:28.053 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.050+0000 7f8ef406b500 10 Cannot find current period zone using local zone 2023-09-22T13:04:28.053 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.050+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.053 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.050+0000 7f8ef406b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:28.053 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.050+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.053 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.051+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:28.054 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.051+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.054 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.051+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:28.054 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.051+0000 7f8ef406b500 20 zone default found 2023-09-22T13:04:28.054 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.051+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.054 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.051+0000 7f8ef406b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:28.054 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.051+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.054 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.052+0000 7f8ef406b500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:28.054 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.052+0000 7f8ef406b500 20 started sync module instance, tier type = 2023-09-22T13:04:28.054 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.052+0000 7f8ef406b500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:28.064 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.063+0000 7f8eded4c700 0 -- 172.21.15.170:0/680322368 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f8ed401f950 msgr2=0x7f8ed400b330 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:28.064 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.063+0000 7f8ef406b500 20 add_watcher() i=0 2023-09-22T13:04:28.064 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.063+0000 7f8ef406b500 20 add_watcher() i=1 2023-09-22T13:04:28.065 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.063+0000 7f8ef406b500 20 add_watcher() i=2 2023-09-22T13:04:28.065 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.063+0000 7f8ef406b500 20 add_watcher() i=3 2023-09-22T13:04:28.065 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.063+0000 7f8ef406b500 20 add_watcher() i=4 2023-09-22T13:04:28.065 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.063+0000 7f8ef406b500 20 add_watcher() i=5 2023-09-22T13:04:28.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.072+0000 7f8ef406b500 20 add_watcher() i=6 2023-09-22T13:04:28.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.072+0000 7f8ef406b500 20 add_watcher() i=7 2023-09-22T13:04:28.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.072+0000 7f8ef406b500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:28.074 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.073+0000 7f8e2bfff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:28.074 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.073+0000 7f8ef406b500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:28.074 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.073+0000 7f8ef406b500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:28.077 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.077+0000 7f8edfd4e700 0 -- 172.21.15.170:0/92951089 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x55d9e6ff86a0 msgr2=0x55d9e6fef9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:28.278 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.278+0000 7f8edfd4e700 0 --2- 172.21.15.170:0/92951089 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x55d9e6ff86a0 0x55d9e6fef9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:04:28.281 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.280+0000 7f8ef406b500 5 note: GC not initialized 2023-09-22T13:04:28.282 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.281+0000 7f8e1affd700 20 reqs_thread_entry: start 2023-09-22T13:04:28.282 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.281+0000 7f8e18ff9700 20 reqs_thread_entry: start 2023-09-22T13:04:28.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.281+0000 7f8ef406b500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:28.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.281+0000 7f8ef406b500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:28.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.281+0000 7f8e12ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:04:58 2023 2023-09-22T13:04:28.283 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:28.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.281+0000 7f8e127fc700 20 reqs_thread_entry: start 2023-09-22T13:04:28.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.281+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc470 obj=default.rgw.meta:root:check-bucket state=0x55d9e6ffa5e0 s->prefetch_data=0 2023-09-22T13:04:28.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.282+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.282+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.286 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.282+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:28.286 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.282+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc470 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.286 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.282+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.286 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.282+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.286 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.282+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.282+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.283+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.283+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.283+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc240 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.283+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.283+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.283+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.283+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.283+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.283+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.283+0000 7f8ef406b500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:28.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.284+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc240 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.284+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.284+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.284+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.284+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.284+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.284+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.289 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.284+0000 7f8ef406b500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:28.289 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.285+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc240 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.292 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.286+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.286+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.286+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.286+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.286+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.286+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.286+0000 7f8ef406b500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:28.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.286+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc240 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.287+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.287+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.287+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.287+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.287+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.287+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.287+0000 7f8ef406b500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:28.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.287+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc240 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.288+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.288+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.288+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.288+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.288+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.288+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.288+0000 7f8ef406b500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:28.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.289+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc240 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.289+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.289+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.289+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.289+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.289+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.289+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.289+0000 7f8ef406b500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:28.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.290+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc240 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.290+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.290+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.290+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.290+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.290+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.290+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.290+0000 7f8ef406b500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:28.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.291+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc240 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.291+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.291+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.291+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.291+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.291+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.291+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.291+0000 7f8ef406b500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:28.298 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.292+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc240 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.298 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.292+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.298 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.292+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.298 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.292+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.298 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.292+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.292+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.292+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.292+0000 7f8ef406b500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:28.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.293+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc240 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.293+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.293+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.293+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.293+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.293+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.293+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.293+0000 7f8ef406b500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:28.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.294+0000 7f8ef406b500 20 get_system_obj_state: rctx=0x7ffe78abc240 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d9e6ee2470 s->prefetch_data=0 2023-09-22T13:04:28.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.294+0000 7f8ef406b500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:28.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.294+0000 7f8ef406b500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:28.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.294+0000 7f8ef406b500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:28.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.294+0000 7f8ef406b500 20 rados->read ofs=0 len=0 2023-09-22T13:04:28.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.294+0000 7f8ef406b500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:28.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.295+0000 7f8ef406b500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:28.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.295+0000 7f8ef406b500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:28.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.296+0000 7f8ef406b500 20 remove_watcher() i=0 2023-09-22T13:04:28.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.296+0000 7f8ef406b500 2 removed watcher, disabling cache 2023-09-22T13:04:28.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.297+0000 7f8ef406b500 20 remove_watcher() i=1 2023-09-22T13:04:28.303 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.298+0000 7f8ef406b500 20 remove_watcher() i=2 2023-09-22T13:04:28.303 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.299+0000 7f8ef406b500 20 remove_watcher() i=3 2023-09-22T13:04:28.303 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.300+0000 7f8ef406b500 20 remove_watcher() i=4 2023-09-22T13:04:28.303 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.301+0000 7f8ef406b500 20 remove_watcher() i=5 2023-09-22T13:04:28.304 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.302+0000 7f8ef406b500 20 remove_watcher() i=6 2023-09-22T13:04:28.306 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.305+0000 7f8ef406b500 20 remove_watcher() i=7 2023-09-22T13:04:28.307 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:28.306+0000 7f8e2bfff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:28.317 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:28.317 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:28.317 INFO:tasks.workunit.client.0.smithi170.stderr:ceph config rm client rgw_debug_inject_set_olh_err 2023-09-22T13:04:28.706 INFO:tasks.workunit.client.0.smithi170.stderr:ceph config rm client rgw_debug_inject_olh_cancel_modification_err 2023-09-22T13:04:29.094 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: bucket check unlinked finds unlistable entries 2023-09-22T13:04:29.094 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.095 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --min-age-hours 0 --dump-keys 2023-09-22T13:04:29.122 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:29.122 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:29.130 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.129+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.132 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.131+0000 7f50b0008500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.132 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.131+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.133 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.132+0000 7f50b0008500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.133 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.132+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.133 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.132+0000 7f50b0008500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.133 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.132+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.133 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.132+0000 7f50b0008500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.144 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.143+0000 7f50b0008500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:29.144 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.143+0000 7f50b0008500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:29.144 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.143+0000 7f50b0008500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:29.144 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.143+0000 7f50b0008500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:29.145 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.143+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.145 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.143+0000 7f50b0008500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.145 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.143+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.145 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.143+0000 7f50b0008500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.145 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.143+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.145 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.143+0000 7f50b0008500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.145 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.143+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.144+0000 7f50b0008500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.144+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.144+0000 7f50b0008500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.144+0000 7f50b0008500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:29.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.144+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.144+0000 7f50b0008500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.144+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.144+0000 7f50b0008500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.149 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.144+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.149 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.145+0000 7f50b0008500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.149 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.145+0000 7f50b0008500 20 zonegroup default 2023-09-22T13:04:29.149 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.145+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.149 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.145+0000 7f50b0008500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.149 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.145+0000 7f50b0008500 10 Cannot find current period zone using local zone 2023-09-22T13:04:29.149 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.145+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.149 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.145+0000 7f50b0008500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.149 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.145+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.145+0000 7f50b0008500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.145+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.146+0000 7f50b0008500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.146+0000 7f50b0008500 20 zone default found 2023-09-22T13:04:29.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.146+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.146+0000 7f50b0008500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.146+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.146+0000 7f50b0008500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.146+0000 7f50b0008500 20 started sync module instance, tier type = 2023-09-22T13:04:29.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.146+0000 7f50b0008500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:29.155 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.155+0000 7f509ace9700 0 -- 172.21.15.170:0/1379699377 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f509004c4f0 msgr2=0x7f509004b5a0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:29.157 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.156+0000 7f50b0008500 20 add_watcher() i=0 2023-09-22T13:04:29.157 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.156+0000 7f50b0008500 20 add_watcher() i=1 2023-09-22T13:04:29.157 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.156+0000 7f50b0008500 20 add_watcher() i=2 2023-09-22T13:04:29.157 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.156+0000 7f50b0008500 20 add_watcher() i=3 2023-09-22T13:04:29.157 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.156+0000 7f50b0008500 20 add_watcher() i=4 2023-09-22T13:04:29.157 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.156+0000 7f50b0008500 20 add_watcher() i=5 2023-09-22T13:04:29.158 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.157+0000 7f50b0008500 20 add_watcher() i=6 2023-09-22T13:04:29.158 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.158+0000 7f50b0008500 20 add_watcher() i=7 2023-09-22T13:04:29.159 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.158+0000 7f50b0008500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:29.161 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.160+0000 7f4fe7fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.161 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.160+0000 7f50b0008500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:29.161 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.160+0000 7f50b0008500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:29.165 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.164+0000 7f509ace9700 0 -- 172.21.15.170:0/3280012418 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x55ea71268190 msgr2=0x55ea7125bc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:29.167 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.166+0000 7f50b0008500 5 note: GC not initialized 2023-09-22T13:04:29.169 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.167+0000 7f4fd6ffd700 20 reqs_thread_entry: start 2023-09-22T13:04:29.169 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.167+0000 7f4fd4ff9700 20 reqs_thread_entry: start 2023-09-22T13:04:29.169 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.167+0000 7f50b0008500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:29.169 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.167+0000 7f50b0008500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:29.169 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f4fceffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:04:59 2023 2023-09-22T13:04:29.170 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.170 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f4fce7fc700 20 reqs_thread_entry: start 2023-09-22T13:04:29.170 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 20 get_system_obj_state: rctx=0x7ffc1c0b6458 obj=default.rgw.meta:root:check-bucket state=0x55ea712012f0 s->prefetch_data=0 2023-09-22T13:04:29.170 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:04:29.170 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:04:29.170 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:29.170 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.170 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.170 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:04:29.171 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.171 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:29.171 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:04:29.171 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:29.171 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 20 get_system_obj_state: rctx=0x7ffc1c0b6528 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55ea712012f0 s->prefetch_data=0 2023-09-22T13:04:29.171 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.168+0000 7f50b0008500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : miss 2023-09-22T13:04:29.173 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x16 2023-09-22T13:04:29.173 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:29.173 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.173 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:29.173 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.174 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.174 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.174 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:04:29.174 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.174 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.174 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x13 2023-09-22T13:04:29.174 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:29.174 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.174 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:29.175 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 2023-09-22T13:04:29.175 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.169+0000 7f50b0008500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:29.175 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.170+0000 7f50b0008500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:04:29.175 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.170+0000 7f50b0008500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.175 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.170+0000 7f50b0008500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:29.175 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.171+0000 7f50b0008500 1 NOTICE: finished shard 2 (0 entries found) 2023-09-22T13:04:29.175 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.171+0000 7f50b0008500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:29.175 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.171+0000 7f50b0008500 1 NOTICE: finished shard 3 (0 entries found) 2023-09-22T13:04:29.175 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.171+0000 7f50b0008500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:29.175 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.171+0000 7f50b0008500 1 NOTICE: finished shard 4 (0 entries found) 2023-09-22T13:04:29.176 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.171+0000 7f50b0008500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.176 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.172+0000 7f50b0008500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:29.176 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.172+0000 7f50b0008500 1 NOTICE: finished shard 6 (0 entries found) 2023-09-22T13:04:29.176 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.172+0000 7f50b0008500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:29.177 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.173+0000 7f50b0008500 1 NOTICE: finished shard 7 (0 entries found) 2023-09-22T13:04:29.177 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.173+0000 7f50b0008500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.177 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.173+0000 7f50b0008500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:29.177 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.174+0000 7f50b0008500 1 NOTICE: finished shard 9 (0 entries found) 2023-09-22T13:04:29.177 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.174+0000 7f50b0008500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.178 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.174+0000 7f50b0008500 1 NOTICE: finished shard 1 (1 entries found) 2023-09-22T13:04:29.178 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.174+0000 7f50b0008500 1 NOTICE: finished shard 5 (1 entries found) 2023-09-22T13:04:29.178 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.174+0000 7f50b0008500 1 NOTICE: finished shard 8 (1 entries found) 2023-09-22T13:04:29.178 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.174+0000 7f50b0008500 1 NOTICE: finished shard 10 (1 entries found) 2023-09-22T13:04:29.178 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.174+0000 7f50b0008500 1 NOTICE: finished all shards (4 entries found) 2023-09-22T13:04:29.178 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.176+0000 7f50b0008500 20 remove_watcher() i=0 2023-09-22T13:04:29.178 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.176+0000 7f50b0008500 2 removed watcher, disabling cache 2023-09-22T13:04:29.179 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.177+0000 7f50b0008500 20 remove_watcher() i=1 2023-09-22T13:04:29.179 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.178+0000 7f50b0008500 20 remove_watcher() i=2 2023-09-22T13:04:29.180 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.179+0000 7f50b0008500 20 remove_watcher() i=3 2023-09-22T13:04:29.181 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.180+0000 7f50b0008500 20 remove_watcher() i=4 2023-09-22T13:04:29.182 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.181+0000 7f50b0008500 20 remove_watcher() i=5 2023-09-22T13:04:29.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.183+0000 7f50b0008500 20 remove_watcher() i=6 2023-09-22T13:04:29.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.187+0000 7f50b0008500 20 remove_watcher() i=7 2023-09-22T13:04:29.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.188+0000 7f4fe7fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.193 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: unlinked entries are not listable 2023-09-22T13:04:29.193 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.202 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.202 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.202 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: GET returns 404 for unlinked entry keys that have no other versions 2023-09-22T13:04:29.202 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.305 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.305 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.305 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.305 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.305 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: bucket check unlinked fixes unlistable entries 2023-09-22T13:04:29.305 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.306 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys 2023-09-22T13:04:29.334 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:29.334 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:29.343 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.342+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.344 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.343+0000 7f764117f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.344 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.343+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.345 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.344+0000 7f764117f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.345 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.344+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.345 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.344+0000 7f764117f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.345 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.344+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.346 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.345+0000 7f764117f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.357 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.356+0000 7f764117f500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:29.357 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.356+0000 7f764117f500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:29.357 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.356+0000 7f764117f500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:29.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.356+0000 7f764117f500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:29.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.356+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.356+0000 7f764117f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.356+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.356+0000 7f764117f500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.356+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.356+0000 7f764117f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.358 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.356+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.361 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.357+0000 7f764117f500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.361 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.357+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.361 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.357+0000 7f764117f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.361 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.357+0000 7f764117f500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:29.361 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.357+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.361 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.357+0000 7f764117f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.362 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.357+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.362 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.358+0000 7f764117f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.362 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.358+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.362 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.358+0000 7f764117f500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.362 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.358+0000 7f764117f500 20 zonegroup default 2023-09-22T13:04:29.362 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.358+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.362 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.358+0000 7f764117f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.363 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.358+0000 7f764117f500 10 Cannot find current period zone using local zone 2023-09-22T13:04:29.363 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.358+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.363 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.358+0000 7f764117f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.363 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.358+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.363 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.359+0000 7f764117f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.363 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.359+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.363 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.359+0000 7f764117f500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.363 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.359+0000 7f764117f500 20 zone default found 2023-09-22T13:04:29.365 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.359+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.365 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.359+0000 7f764117f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.365 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.359+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.365 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.360+0000 7f764117f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.366 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.360+0000 7f764117f500 20 started sync module instance, tier type = 2023-09-22T13:04:29.366 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.360+0000 7f764117f500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:29.366 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.366+0000 7f762be60700 0 -- 172.21.15.170:0/2255160596 >> [v2:172.21.15.170:6800/145025,v1:172.21.15.170:6801/145025] conn(0x7f761806b860 msgr2=0x7f761806dc90 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:29.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.375+0000 7f764117f500 20 add_watcher() i=0 2023-09-22T13:04:29.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.375+0000 7f764117f500 20 add_watcher() i=1 2023-09-22T13:04:29.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.375+0000 7f764117f500 20 add_watcher() i=2 2023-09-22T13:04:29.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.375+0000 7f764117f500 20 add_watcher() i=3 2023-09-22T13:04:29.376 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.375+0000 7f764117f500 20 add_watcher() i=4 2023-09-22T13:04:29.377 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.375+0000 7f764117f500 20 add_watcher() i=5 2023-09-22T13:04:29.377 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.375+0000 7f764117f500 20 add_watcher() i=6 2023-09-22T13:04:29.378 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.377+0000 7f764117f500 20 add_watcher() i=7 2023-09-22T13:04:29.378 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.377+0000 7f764117f500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:29.380 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.379+0000 7f75fdfe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.380 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.380+0000 7f764117f500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:29.380 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.380+0000 7f764117f500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:29.384 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.383+0000 7f762ce62700 0 -- >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x55d574f7fb70 msgr2=0x55d574f83670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:29.386 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.385+0000 7f764117f500 5 note: GC not initialized 2023-09-22T13:04:29.388 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.386+0000 7f7574ff9700 20 reqs_thread_entry: start 2023-09-22T13:04:29.388 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.386+0000 7f756a7fc700 20 reqs_thread_entry: start 2023-09-22T13:04:29.388 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.387+0000 7f764117f500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:29.388 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.387+0000 7f764117f500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:29.388 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.387+0000 7f7568ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:04:59 2023 2023-09-22T13:04:29.388 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.389 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.387+0000 7f754ffff700 20 reqs_thread_entry: start 2023-09-22T13:04:29.389 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.387+0000 7f764117f500 20 get_system_obj_state: rctx=0x7ffd696643f8 obj=default.rgw.meta:root:check-bucket state=0x55d574f26180 s->prefetch_data=0 2023-09-22T13:04:29.389 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.387+0000 7f764117f500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:04:29.392 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:04:29.392 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:29.392 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.392 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.392 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:04:29.392 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.392 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:29.393 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:04:29.393 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:29.393 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 20 get_system_obj_state: rctx=0x7ffd696644c8 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d574f26180 s->prefetch_data=0 2023-09-22T13:04:29.393 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : miss 2023-09-22T13:04:29.393 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x16 2023-09-22T13:04:29.393 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:29.393 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.394 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:29.394 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.394 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.394 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.394 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:04:29.394 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.388+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.394 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.389+0000 7f764117f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.395 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.389+0000 7f764117f500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x13 2023-09-22T13:04:29.395 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.389+0000 7f764117f500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:29.395 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.389+0000 7f764117f500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.395 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.389+0000 7f764117f500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:29.395 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.389+0000 7f764117f500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 2023-09-22T13:04:29.395 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.389+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:29.395 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.389+0000 7f764117f500 1 NOTICE: finished shard 0 (0 entries removed) 2023-09-22T13:04:29.395 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.389+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.390+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:29.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.390+0000 7f764117f500 1 NOTICE: finished shard 2 (0 entries removed) 2023-09-22T13:04:29.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.390+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:29.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.391+0000 7f764117f500 1 NOTICE: finished shard 3 (0 entries removed) 2023-09-22T13:04:29.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.391+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:29.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.391+0000 7f764117f500 1 NOTICE: finished shard 4 (0 entries removed) 2023-09-22T13:04:29.396 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.391+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.398 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.392+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:29.398 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.392+0000 7f764117f500 1 NOTICE: finished shard 6 (0 entries removed) 2023-09-22T13:04:29.398 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.392+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:29.398 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.392+0000 7f764117f500 1 NOTICE: finished shard 7 (0 entries removed) 2023-09-22T13:04:29.398 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.392+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.398 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.393+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:29.398 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.393+0000 7f764117f500 1 NOTICE: finished shard 9 (0 entries removed) 2023-09-22T13:04:29.399 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.393+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.399 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.394+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.399 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.394+0000 7f764117f500 20 get_obj_state: rctx=0x55d575027d70 obj=check-bucket:e state=0x55d57502b768 s->prefetch_data=0 2023-09-22T13:04:29.399 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.394+0000 7f764117f500 20 get_obj_state: setting s->obj_tag to 5ajhkm349vdwkrvcutzinxdm0dxrs015 2023-09-22T13:04:29.399 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.394+0000 7f764117f500 20 get_obj_state_impl: setting s->olh_tag to 0zzi8cnltoo3sh65thfobhdtepi0mfe3 2023-09-22T13:04:29.399 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.394+0000 7f764117f500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=0zzi8cnltoo3sh65thfobhdtepi0mfe3 2023-09-22T13:04:29.399 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.396+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.401+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.401+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.401+0000 7f764117f500 20 olh_log_entry: epoch=2 op=2 key=e[] 2023-09-22T13:04:29.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.401+0000 7f764117f500 20 olh_log_entry: epoch=2 op=3 key=e[k3PfRYuUabgyNqvvoUKk9ahGo6iXwFr] 2023-09-22T13:04:29.402 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.401+0000 7f764117f500 20 get_obj_state: rctx=0x55d575027d70 obj=check-bucket:_:k3PfRYuUabgyNqvvoUKk9ahGo6iXwFr_e state=0x55d575096428 s->prefetch_data=0 2023-09-22T13:04:29.403 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.402+0000 7f764117f500 10 manifest: total_size = 22 2023-09-22T13:04:29.403 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.402+0000 7f764117f500 20 get_obj_state: setting s->obj_tag to 772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4689.11065865148128625546 2023-09-22T13:04:29.403 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.402+0000 7f764117f500 20 get_obj_state: rctx=0x55d575027d70 obj=check-bucket:_:k3PfRYuUabgyNqvvoUKk9ahGo6iXwFr_e state=0x55d575096428 s->prefetch_data=0 2023-09-22T13:04:29.404 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.402+0000 7f764117f500 20 prepare_atomic_modification: state is not atomic. state=0x55d575096428 2023-09-22T13:04:29.404 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.402+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.406 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.404+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : miss 2023-09-22T13:04:29.406 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.404+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.406 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.405+0000 7f764117f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.406 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.405+0000 7f764117f500 10 cache put: name=default.rgw.log++bucket.sync-source-hints.check-bucket info.flags=0x0 2023-09-22T13:04:29.406 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.405+0000 7f764117f500 10 adding default.rgw.log++bucket.sync-source-hints.check-bucket to cache LRU end 2023-09-22T13:04:29.406 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.405+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : miss 2023-09-22T13:04:29.406 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.405+0000 7f764117f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.407 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.405+0000 7f764117f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.407 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.405+0000 7f764117f500 10 cache put: name=default.rgw.log++bucket.sync-target-hints.check-bucket info.flags=0x0 2023-09-22T13:04:29.407 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.405+0000 7f764117f500 10 adding default.rgw.log++bucket.sync-target-hints.check-bucket to cache LRU end 2023-09-22T13:04:29.407 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.405+0000 7f764117f500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:29.407 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.405+0000 7f764117f500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:29.407 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.405+0000 7f764117f500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:29.409 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.408+0000 7f764117f500 20 get_obj_state: rctx=0x55d575027d70 obj=check-bucket:e state=0x55d57502b768 s->prefetch_data=0 2023-09-22T13:04:29.409 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.408+0000 7f764117f500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=0zzi8cnltoo3sh65thfobhdtepi0mfe3 2023-09-22T13:04:29.418 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.417+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.420 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.419+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.421 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.419+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.421 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.419+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.421 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.419+0000 7f764117f500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:29.421 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.419+0000 7f764117f500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:29.421 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.419+0000 7f764117f500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:29.421 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.419+0000 7f764117f500 1 NOTICE: finished shard 1 (1 entries removed) 2023-09-22T13:04:29.421 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.419+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.421 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.419+0000 7f764117f500 20 get_obj_state: rctx=0x55d57504c070 obj=check-bucket:d state=0x55d57502b6e8 s->prefetch_data=0 2023-09-22T13:04:29.422 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.420+0000 7f764117f500 20 get_obj_state: setting s->obj_tag to qj18wi4cgp7z2ph3fwk2w8peez1vmhfc 2023-09-22T13:04:29.422 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.420+0000 7f764117f500 20 get_obj_state_impl: setting s->olh_tag to 1ormwvc1x16rnao5v4juvavrjcc1xaiu 2023-09-22T13:04:29.422 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.420+0000 7f764117f500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=1ormwvc1x16rnao5v4juvavrjcc1xaiu 2023-09-22T13:04:29.422 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.421+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.423 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.422+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.424 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.422+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.424 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.423+0000 7f764117f500 20 olh_log_entry: epoch=3 op=3 key=d[Ab8zsIeGtGJ04W86.lcUAxCoHF6TAjb] 2023-09-22T13:04:29.424 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.423+0000 7f764117f500 20 get_obj_state: rctx=0x55d57504c070 obj=check-bucket:_:Ab8zsIeGtGJ04W86.lcUAxCoHF6TAjb_d state=0x55d575095418 s->prefetch_data=0 2023-09-22T13:04:29.424 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.423+0000 7f764117f500 10 manifest: total_size = 22 2023-09-22T13:04:29.424 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.423+0000 7f764117f500 20 get_obj_state: setting s->obj_tag to 772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4689.6698258469177934488 2023-09-22T13:04:29.424 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.423+0000 7f764117f500 20 get_obj_state: rctx=0x55d57504c070 obj=check-bucket:_:Ab8zsIeGtGJ04W86.lcUAxCoHF6TAjb_d state=0x55d575095418 s->prefetch_data=0 2023-09-22T13:04:29.424 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.423+0000 7f764117f500 20 prepare_atomic_modification: state is not atomic. state=0x55d575095418 2023-09-22T13:04:29.425 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.423+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.426 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.425+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.426 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.425+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.426 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.425+0000 7f764117f500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:29.426 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.425+0000 7f764117f500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:29.426 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.425+0000 7f764117f500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:29.428 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.427+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.428 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.427+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.428+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.428+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.430 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.428+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.430 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.428+0000 7f764117f500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:29.430 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.428+0000 7f764117f500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:29.430 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.428+0000 7f764117f500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:29.430 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.428+0000 7f764117f500 1 NOTICE: finished shard 5 (1 entries removed) 2023-09-22T13:04:29.430 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.428+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.430 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.428+0000 7f764117f500 20 get_obj_state: rctx=0x55d57506cb70 obj=check-bucket:c state=0x55d57502b6e8 s->prefetch_data=0 2023-09-22T13:04:29.430 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.429+0000 7f764117f500 20 get_obj_state: setting s->obj_tag to d2l5h49vf0eahw30o1omgem89lcc7dwq 2023-09-22T13:04:29.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.429+0000 7f764117f500 20 get_obj_state_impl: setting s->olh_tag to 7q00u7gs97dsxej1yly67972ei7mssbp 2023-09-22T13:04:29.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.429+0000 7f764117f500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=7q00u7gs97dsxej1yly67972ei7mssbp 2023-09-22T13:04:29.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.430+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.432 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.431+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.432 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.431+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.433 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.432+0000 7f764117f500 20 olh_log_entry: epoch=3 op=3 key=c[H740L-ZGlDTDcPsdy64IQO73PwtrWR3] 2023-09-22T13:04:29.433 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.432+0000 7f764117f500 20 get_obj_state: rctx=0x55d57506cb70 obj=check-bucket:_:H740L-ZGlDTDcPsdy64IQO73PwtrWR3_c state=0x55d575095418 s->prefetch_data=0 2023-09-22T13:04:29.433 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.432+0000 7f764117f500 10 manifest: total_size = 22 2023-09-22T13:04:29.433 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.432+0000 7f764117f500 20 get_obj_state: setting s->obj_tag to 772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4689.13491875538474421491 2023-09-22T13:04:29.434 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.432+0000 7f764117f500 20 get_obj_state: rctx=0x55d57506cb70 obj=check-bucket:_:H740L-ZGlDTDcPsdy64IQO73PwtrWR3_c state=0x55d575095418 s->prefetch_data=0 2023-09-22T13:04:29.434 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.432+0000 7f764117f500 20 prepare_atomic_modification: state is not atomic. state=0x55d575095418 2023-09-22T13:04:29.434 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.432+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.435 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.435+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.436 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.435+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.436 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.435+0000 7f764117f500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:29.436 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.435+0000 7f764117f500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:29.436 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.435+0000 7f764117f500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:29.437 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.436+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.437 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.436+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.439 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.437+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.439 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.437+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.439 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.437+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.439 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.437+0000 7f764117f500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:29.439 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.437+0000 7f764117f500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:29.439 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.437+0000 7f764117f500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:29.439 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.437+0000 7f764117f500 1 NOTICE: finished shard 8 (1 entries removed) 2023-09-22T13:04:29.439 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.437+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.439 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.438+0000 7f764117f500 20 get_obj_state: rctx=0x55d57508f370 obj=check-bucket:f state=0x55d57502b6a8 s->prefetch_data=0 2023-09-22T13:04:29.440 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.438+0000 7f764117f500 20 get_obj_state: setting s->obj_tag to dhoo1kryt3lioutltp6boy5r5n5d3ect 2023-09-22T13:04:29.440 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.438+0000 7f764117f500 20 get_obj_state_impl: setting s->olh_tag to i070p6xbbfnvfdfoq5ebiak9zqfwmq4r 2023-09-22T13:04:29.440 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.438+0000 7f764117f500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=i070p6xbbfnvfdfoq5ebiak9zqfwmq4r 2023-09-22T13:04:29.440 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.439+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.441 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.441+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.442 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.441+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.442 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.441+0000 7f764117f500 20 olh_log_entry: epoch=2 op=2 key=f[] 2023-09-22T13:04:29.442 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.441+0000 7f764117f500 20 olh_log_entry: epoch=2 op=3 key=f[5M1i5PA.GAcz3v3K.ofFOL52ItEE8Pv] 2023-09-22T13:04:29.442 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.441+0000 7f764117f500 20 get_obj_state: rctx=0x55d57508f370 obj=check-bucket:_:5M1i5PA.GAcz3v3K.ofFOL52ItEE8Pv_f state=0x55d575097c18 s->prefetch_data=0 2023-09-22T13:04:29.442 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.441+0000 7f764117f500 10 manifest: total_size = 11 2023-09-22T13:04:29.443 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.441+0000 7f764117f500 20 get_obj_state: setting s->obj_tag to 772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4689.2406287482041678194 2023-09-22T13:04:29.443 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.441+0000 7f764117f500 20 get_obj_state: rctx=0x55d57508f370 obj=check-bucket:_:5M1i5PA.GAcz3v3K.ofFOL52ItEE8Pv_f state=0x55d575097c18 s->prefetch_data=0 2023-09-22T13:04:29.443 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.441+0000 7f764117f500 20 prepare_atomic_modification: state is not atomic. state=0x55d575097c18 2023-09-22T13:04:29.443 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.441+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.445 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.444+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.445 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.444+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.445 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.444+0000 7f764117f500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:29.445 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.444+0000 7f764117f500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:29.445 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.444+0000 7f764117f500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:29.446 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.445+0000 7f764117f500 20 get_obj_state: rctx=0x55d57508f370 obj=check-bucket:f state=0x55d57502b6a8 s->prefetch_data=0 2023-09-22T13:04:29.446 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.445+0000 7f764117f500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=i070p6xbbfnvfdfoq5ebiak9zqfwmq4r 2023-09-22T13:04:29.449 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.448+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.451 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.450+0000 7f764117f500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.451 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.450+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.451 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.450+0000 7f764117f500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:29.452 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.450+0000 7f764117f500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:29.452 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.450+0000 7f764117f500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:29.452 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.450+0000 7f764117f500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:29.452 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.450+0000 7f764117f500 1 NOTICE: finished shard 10 (1 entries removed) 2023-09-22T13:04:29.452 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.450+0000 7f764117f500 1 NOTICE: finished all shards (4 entries removed) 2023-09-22T13:04:29.453 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.451+0000 7f764117f500 20 remove_watcher() i=0 2023-09-22T13:04:29.453 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.451+0000 7f764117f500 2 removed watcher, disabling cache 2023-09-22T13:04:29.453 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.452+0000 7f764117f500 20 remove_watcher() i=1 2023-09-22T13:04:29.454 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.453+0000 7f764117f500 20 remove_watcher() i=2 2023-09-22T13:04:29.455 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.455+0000 7f764117f500 20 remove_watcher() i=3 2023-09-22T13:04:29.456 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.456+0000 7f764117f500 20 remove_watcher() i=4 2023-09-22T13:04:29.457 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.457+0000 7f764117f500 20 remove_watcher() i=5 2023-09-22T13:04:29.458 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.458+0000 7f764117f500 20 remove_watcher() i=6 2023-09-22T13:04:29.459 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.459+0000 7f764117f500 20 remove_watcher() i=7 2023-09-22T13:04:29.460 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.459+0000 7f75fdfe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.476 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.476 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.476 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.476 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.476 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.476 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.476 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.476 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.476 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: bucket check unlinked does not affect normal entries 2023-09-22T13:04:29.477 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.532 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.533 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.533 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.533 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.533 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.533 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.533 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.533 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.533 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.534 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.534 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: bucket check unlinked does not find new unlistable entries after fix 2023-09-22T13:04:29.534 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.534 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --min-age-hours 0 --dump-keys 2023-09-22T13:04:29.559 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:29.560 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:29.567 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.566+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.568+0000 7fb670464500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.568+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.570 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.568+0000 7fb670464500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.570 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.568+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.570 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.569+0000 7fb670464500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.570 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.569+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.570 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.569+0000 7fb670464500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.580 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.579+0000 7fb670464500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:29.580 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.579+0000 7fb670464500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:29.612 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.579+0000 7fb670464500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:29.612 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.579+0000 7fb670464500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:29.612 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.579+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.612 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.580+0000 7fb670464500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.612 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.580+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.612 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.580+0000 7fb670464500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.613 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.580+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.613 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.580+0000 7fb670464500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.613 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.580+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.613 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.581+0000 7fb670464500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.614 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.581+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.614 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.581+0000 7fb670464500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.614 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.581+0000 7fb670464500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:29.614 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.581+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.614 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.581+0000 7fb670464500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.614 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.581+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.581+0000 7fb670464500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.581+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.582+0000 7fb670464500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.582+0000 7fb670464500 20 zonegroup default 2023-09-22T13:04:29.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.582+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.582+0000 7fb670464500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.582+0000 7fb670464500 10 Cannot find current period zone using local zone 2023-09-22T13:04:29.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.582+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.582+0000 7fb670464500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.616 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.582+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.616 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.582+0000 7fb670464500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.616 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.582+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.616 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.583+0000 7fb670464500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.617 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.583+0000 7fb670464500 20 zone default found 2023-09-22T13:04:29.617 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.583+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.617 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.583+0000 7fb670464500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.617 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.583+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.618 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.583+0000 7fb670464500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.618 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.583+0000 7fb670464500 20 started sync module instance, tier type = 2023-09-22T13:04:29.618 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.583+0000 7fb670464500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:29.618 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.592+0000 7fb65b145700 0 -- 172.21.15.170:0/764726280 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7fb64401f300 msgr2=0x7fb64404b360 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:29.619 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.594+0000 7fb670464500 20 add_watcher() i=0 2023-09-22T13:04:29.619 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.594+0000 7fb670464500 20 add_watcher() i=1 2023-09-22T13:04:29.619 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.594+0000 7fb670464500 20 add_watcher() i=2 2023-09-22T13:04:29.619 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.594+0000 7fb670464500 20 add_watcher() i=3 2023-09-22T13:04:29.619 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.594+0000 7fb670464500 20 add_watcher() i=4 2023-09-22T13:04:29.620 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.594+0000 7fb670464500 20 add_watcher() i=5 2023-09-22T13:04:29.620 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.594+0000 7fb670464500 20 add_watcher() i=6 2023-09-22T13:04:29.620 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.597+0000 7fb670464500 20 add_watcher() i=7 2023-09-22T13:04:29.620 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.597+0000 7fb670464500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:29.621 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.599+0000 7fb62d7e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.621 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.599+0000 7fb670464500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:29.621 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.599+0000 7fb670464500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:29.621 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.603+0000 7fb65b145700 0 -- 172.21.15.170:0/3957630971 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x5567cb9b61a0 msgr2=0x5567cb9c1100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:29.621 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.605+0000 7fb670464500 5 note: GC not initialized 2023-09-22T13:04:29.622 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.606+0000 7fb59bfff700 20 reqs_thread_entry: start 2023-09-22T13:04:29.622 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.606+0000 7fb599ffb700 20 reqs_thread_entry: start 2023-09-22T13:04:29.622 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.606+0000 7fb670464500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:29.622 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.607+0000 7fb670464500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:29.623 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.607+0000 7fb587fff700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:04:59 2023 2023-09-22T13:04:29.623 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.623 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.607+0000 7fb5877fe700 20 reqs_thread_entry: start 2023-09-22T13:04:29.623 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.607+0000 7fb670464500 20 get_system_obj_state: rctx=0x7ffc55ace338 obj=default.rgw.meta:root:check-bucket state=0x5567cb907ef0 s->prefetch_data=0 2023-09-22T13:04:29.623 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.607+0000 7fb670464500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:04:29.624 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.607+0000 7fb670464500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:04:29.624 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.607+0000 7fb670464500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:29.624 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.607+0000 7fb670464500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.624 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.607+0000 7fb670464500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.625 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.607+0000 7fb670464500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:04:29.625 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.607+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.625 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:29.625 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:04:29.625 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:29.626 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 20 get_system_obj_state: rctx=0x7ffc55ace408 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5567cb907ef0 s->prefetch_data=0 2023-09-22T13:04:29.626 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : miss 2023-09-22T13:04:29.626 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x16 2023-09-22T13:04:29.626 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:29.626 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.626 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:29.626 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.626 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.627 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.627 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:04:29.627 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.627 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.627 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x13 2023-09-22T13:04:29.627 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:29.627 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.627 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:29.628 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 2023-09-22T13:04:29.628 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.608+0000 7fb670464500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:29.628 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.609+0000 7fb670464500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:04:29.628 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.609+0000 7fb670464500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.628 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.610+0000 7fb670464500 1 NOTICE: finished shard 1 (0 entries found) 2023-09-22T13:04:29.628 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.610+0000 7fb670464500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:29.628 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.610+0000 7fb670464500 1 NOTICE: finished shard 2 (0 entries found) 2023-09-22T13:04:29.628 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.610+0000 7fb670464500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:29.628 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.610+0000 7fb670464500 1 NOTICE: finished shard 3 (0 entries found) 2023-09-22T13:04:29.628 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.610+0000 7fb670464500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:29.629 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.611+0000 7fb670464500 1 NOTICE: finished shard 4 (0 entries found) 2023-09-22T13:04:29.629 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.611+0000 7fb670464500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.629 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.611+0000 7fb670464500 1 NOTICE: finished shard 5 (0 entries found) 2023-09-22T13:04:29.629 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.611+0000 7fb670464500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:29.631 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.612+0000 7fb670464500 1 NOTICE: finished shard 6 (0 entries found) 2023-09-22T13:04:29.631 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.612+0000 7fb670464500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:29.631 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.612+0000 7fb670464500 1 NOTICE: finished shard 7 (0 entries found) 2023-09-22T13:04:29.631 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.612+0000 7fb670464500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.632 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.613+0000 7fb670464500 1 NOTICE: finished shard 8 (0 entries found) 2023-09-22T13:04:29.632 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.613+0000 7fb670464500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:29.632 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.613+0000 7fb670464500 1 NOTICE: finished shard 9 (0 entries found) 2023-09-22T13:04:29.632 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.613+0000 7fb670464500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.632 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.614+0000 7fb670464500 1 NOTICE: finished shard 10 (0 entries found) 2023-09-22T13:04:29.632 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.614+0000 7fb670464500 1 NOTICE: finished all shards (0 entries found) 2023-09-22T13:04:29.632 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.615+0000 7fb670464500 20 remove_watcher() i=0 2023-09-22T13:04:29.632 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.615+0000 7fb670464500 2 removed watcher, disabling cache 2023-09-22T13:04:29.632 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.616+0000 7fb670464500 20 remove_watcher() i=1 2023-09-22T13:04:29.632 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.618+0000 7fb670464500 20 remove_watcher() i=2 2023-09-22T13:04:29.633 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.619+0000 7fb670464500 20 remove_watcher() i=3 2023-09-22T13:04:29.633 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.620+0000 7fb670464500 20 remove_watcher() i=4 2023-09-22T13:04:29.633 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.621+0000 7fb670464500 20 remove_watcher() i=5 2023-09-22T13:04:29.633 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.623+0000 7fb670464500 20 remove_watcher() i=6 2023-09-22T13:04:29.633 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.626+0000 7fb670464500 20 remove_watcher() i=7 2023-09-22T13:04:29.633 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.627+0000 7fb62d7e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.661 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.661 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.661 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:29.661 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:29.662 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:04:29.685 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:29.685 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:29.693 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.692+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.695 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.694+0000 7feb5625f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.695 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.694+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.696 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.695+0000 7feb5625f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.696 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.695+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.696 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.695+0000 7feb5625f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.696 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.695+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.696 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.696+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.708 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.705+0000 7feb5625f500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:29.708 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.705+0000 7feb5625f500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:29.708 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.705+0000 7feb5625f500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:29.708 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.705+0000 7feb5625f500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:29.708 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.705+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.708 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.705+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.708 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.705+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.708 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.706+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.708 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.706+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.709 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.706+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.709 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.706+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.709 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.706+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.709 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.706+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.709 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.707+0000 7feb5625f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.709 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.707+0000 7feb5625f500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:29.709 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.707+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.709 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.707+0000 7feb5625f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.709 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.707+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.709 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.707+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.710 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.707+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.711 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.707+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.711 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.707+0000 7feb5625f500 20 zonegroup default 2023-09-22T13:04:29.711 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.707+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.711 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.708+0000 7feb5625f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.712 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.708+0000 7feb5625f500 10 Cannot find current period zone using local zone 2023-09-22T13:04:29.712 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.708+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.712 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.708+0000 7feb5625f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.712 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.708+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.712 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.708+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.712 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.708+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.712 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.708+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.712 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.708+0000 7feb5625f500 20 zone default found 2023-09-22T13:04:29.712 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.708+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.712 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.709+0000 7feb5625f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.712 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.709+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.713 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.709+0000 7feb5625f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.713 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.709+0000 7feb5625f500 20 started sync module instance, tier type = 2023-09-22T13:04:29.713 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.709+0000 7feb5625f500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:29.728 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.728+0000 7feb5625f500 20 add_watcher() i=0 2023-09-22T13:04:29.729 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.728+0000 7feb5625f500 20 add_watcher() i=1 2023-09-22T13:04:29.729 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.728+0000 7feb5625f500 20 add_watcher() i=2 2023-09-22T13:04:29.729 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.728+0000 7feb5625f500 20 add_watcher() i=3 2023-09-22T13:04:29.729 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.728+0000 7feb5625f500 20 add_watcher() i=4 2023-09-22T13:04:29.729 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.728+0000 7feb5625f500 20 add_watcher() i=5 2023-09-22T13:04:29.729 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.728+0000 7feb5625f500 20 add_watcher() i=6 2023-09-22T13:04:29.731 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.731+0000 7feb5625f500 20 add_watcher() i=7 2023-09-22T13:04:29.731 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.731+0000 7feb5625f500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:29.732 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.731+0000 7feb41f42700 0 -- 172.21.15.170:0/761283428 >> [v2:172.21.15.191:6816/144992,v1:172.21.15.191:6817/144992] conn(0x7feb34012390 msgr2=0x7feb340127b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:29.734 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.733+0000 7feb127e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.734 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.733+0000 7feb5625f500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:29.734 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.733+0000 7feb5625f500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:29.738 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.737+0000 7feb41f42700 0 -- 172.21.15.170:0/4005365012 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x55a72a96f340 msgr2=0x55a72a96fc70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:29.740 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.740+0000 7feb5625f500 5 note: GC not initialized 2023-09-22T13:04:29.742 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.741+0000 7fea817fa700 20 reqs_thread_entry: start 2023-09-22T13:04:29.742 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.741+0000 7fea7affd700 20 reqs_thread_entry: start 2023-09-22T13:04:29.742 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.741+0000 7feb5625f500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:29.742 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.741+0000 7feb5625f500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:29.743 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.741+0000 7fea797fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:04:59 2023 2023-09-22T13:04:29.743 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.743 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.741+0000 7fea78ff9700 20 reqs_thread_entry: start 2023-09-22T13:04:29.743 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.741+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1c080 obj=default.rgw.meta:root:check-bucket state=0x55a72aa14990 s->prefetch_data=0 2023-09-22T13:04:29.743 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.741+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.743 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.741+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.745 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.742+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:29.745 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.742+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1c080 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.745 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.742+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.745 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.742+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.746 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.742+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.746 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.742+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.746 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.743+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.746 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.743+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.746 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.743+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1be50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.746 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.743+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.746 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.743+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.746 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.743+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.746 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.743+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.746 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.743+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.747 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.743+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.747 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.743+0000 7feb5625f500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:29.747 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.744+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1be50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.747 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.744+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.747 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.744+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.747 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.744+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.747 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.744+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.747 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.744+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.747 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.744+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.747 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.744+0000 7feb5625f500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.750 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.745+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1be50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.750 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.745+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.750 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.745+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.750 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.745+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.751 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.746+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.751 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.746+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.751 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.746+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.751 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.746+0000 7feb5625f500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:29.751 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.746+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1be50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.751 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.747+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.751 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.747+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.751 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.747+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.751 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.747+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.751 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.747+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.752 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.747+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.752 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.747+0000 7feb5625f500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:29.752 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.748+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1be50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.752 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.748+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.752 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.748+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.752 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.748+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.752 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.748+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.752 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.748+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.752 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.748+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.753 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.748+0000 7feb5625f500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:29.753 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.749+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1be50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.753 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.749+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.753 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.749+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.753 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.749+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.753 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.749+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.753 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.749+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.753 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.749+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.754 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.749+0000 7feb5625f500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.757 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.750+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1be50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.757 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.750+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.757 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.750+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.757 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.750+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.757 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.750+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.757 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.750+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.758 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.750+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.758 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.750+0000 7feb5625f500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:29.758 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.751+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1be50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.758 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.751+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.758 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.751+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.758 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.751+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.758 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.751+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.758 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.751+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.758 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.751+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.751+0000 7feb5625f500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:29.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.752+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1be50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.752+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.752+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.752+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.752+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.753+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.753+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.753+0000 7feb5625f500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.753+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1be50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.759 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.753+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.753+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.753+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.753+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.754+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.754+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.754+0000 7feb5625f500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:29.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.754+0000 7feb5625f500 20 get_system_obj_state: rctx=0x7fffe7c1be50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55a72a99cb60 s->prefetch_data=0 2023-09-22T13:04:29.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.754+0000 7feb5625f500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.754+0000 7feb5625f500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.760 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.754+0000 7feb5625f500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.761 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.754+0000 7feb5625f500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.761 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.755+0000 7feb5625f500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.761 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.755+0000 7feb5625f500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.761 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.755+0000 7feb5625f500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.761 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.756+0000 7feb5625f500 20 remove_watcher() i=0 2023-09-22T13:04:29.761 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.756+0000 7feb5625f500 2 removed watcher, disabling cache 2023-09-22T13:04:29.762 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.758+0000 7feb5625f500 20 remove_watcher() i=1 2023-09-22T13:04:29.762 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.761+0000 7feb5625f500 20 remove_watcher() i=2 2023-09-22T13:04:29.762 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.762+0000 7feb5625f500 20 remove_watcher() i=3 2023-09-22T13:04:29.763 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.763+0000 7feb5625f500 20 remove_watcher() i=4 2023-09-22T13:04:29.764 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.764+0000 7feb5625f500 20 remove_watcher() i=5 2023-09-22T13:04:29.770 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.769+0000 7feb5625f500 20 remove_watcher() i=6 2023-09-22T13:04:29.771 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.770+0000 7feb5625f500 20 remove_watcher() i=7 2023-09-22T13:04:29.771 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.771+0000 7feb127e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.777 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:04:29.802 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:29.802 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:29.809 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.809+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.811 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.810+0000 7f7c227cb500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.811 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.810+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.812 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.811+0000 7f7c227cb500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.812 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.811+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.812 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.811+0000 7f7c227cb500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.812 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.811+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.813 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.812+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.823 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.822+0000 7f7c227cb500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:29.824 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.822+0000 7f7c227cb500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:29.824 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.822+0000 7f7c227cb500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:29.824 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.822+0000 7f7c227cb500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:29.824 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.822+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.824 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.822+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.824 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.822+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.824 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.822+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.824 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.822+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.824 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.823+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.825 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.823+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.827 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.823+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.827 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.823+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.827 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.823+0000 7f7c227cb500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.827 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.823+0000 7f7c227cb500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:29.827 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.823+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.827 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.827 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.828 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.828 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.828 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.828 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 20 zonegroup default 2023-09-22T13:04:29.828 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.828 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.828 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 10 Cannot find current period zone using local zone 2023-09-22T13:04:29.828 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.828 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.828 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.824+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.825+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.825+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.825+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.825+0000 7f7c227cb500 20 zone default found 2023-09-22T13:04:29.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.825+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.825+0000 7f7c227cb500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.825+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.825+0000 7f7c227cb500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.826+0000 7f7c227cb500 20 started sync module instance, tier type = 2023-09-22T13:04:29.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.826+0000 7f7c227cb500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:29.841 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.841+0000 7f7c227cb500 20 add_watcher() i=0 2023-09-22T13:04:29.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.841+0000 7f7c227cb500 20 add_watcher() i=1 2023-09-22T13:04:29.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.841+0000 7f7c227cb500 20 add_watcher() i=2 2023-09-22T13:04:29.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.841+0000 7f7c227cb500 20 add_watcher() i=3 2023-09-22T13:04:29.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.841+0000 7f7c227cb500 20 add_watcher() i=4 2023-09-22T13:04:29.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.841+0000 7f7c227cb500 20 add_watcher() i=5 2023-09-22T13:04:29.844 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.843+0000 7f7c0d4ac700 0 -- 172.21.15.170:0/333238783 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f7c04025950 msgr2=0x7f7c040233c0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:29.844 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.844+0000 7f7c227cb500 20 add_watcher() i=6 2023-09-22T13:04:29.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.845+0000 7f7c227cb500 20 add_watcher() i=7 2023-09-22T13:04:29.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.845+0000 7f7c227cb500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:29.847 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.847+0000 7f7bdf7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.848 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.847+0000 7f7c227cb500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:29.848 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.847+0000 7f7c227cb500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:29.851 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.851+0000 7f7c0dcad700 0 -- 172.21.15.170:0/3412461484 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x563dfbf66690 msgr2=0x563dfbf5e8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:29.853 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.853+0000 7f7c227cb500 5 note: GC not initialized 2023-09-22T13:04:29.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.853+0000 7f7b4e7fc700 20 reqs_thread_entry: start 2023-09-22T13:04:29.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.854+0000 7f7b3ffff700 20 reqs_thread_entry: start 2023-09-22T13:04:29.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.854+0000 7f7c227cb500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:29.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.854+0000 7f7c227cb500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:29.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.854+0000 7f7b3e7fc700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:00 2023 2023-09-22T13:04:29.855 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.855 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.854+0000 7f7b3dffb700 20 reqs_thread_entry: start 2023-09-22T13:04:29.856 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.854+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7ecd0 obj=default.rgw.meta:root:check-bucket state=0x563dfbed9af0 s->prefetch_data=0 2023-09-22T13:04:29.858 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.855+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.858 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.855+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.858 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.855+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:29.858 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.855+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7ecd0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.858 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.855+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.858 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.855+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.858 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.855+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.858 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.855+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.859 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.856+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.859 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.856+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.859 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.856+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7eaa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.859 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.856+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.859 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.856+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.859 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.856+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.859 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.856+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.859 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.856+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.859 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.856+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.859 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.856+0000 7f7c227cb500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:29.860 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.857+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7eaa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.860 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.857+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.860 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.857+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.860 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.857+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.860 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.857+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.860 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.857+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.860 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.857+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.860 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.857+0000 7f7c227cb500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.863 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.858+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7eaa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.863 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.858+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.863 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.858+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.863 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.858+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.863 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.858+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.863 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.858+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.863 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.858+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.858+0000 7f7c227cb500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:29.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.859+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7eaa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.859+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.859+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.859+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.859+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.860+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.860+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.864 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.860+0000 7f7c227cb500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:29.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.860+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7eaa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.861+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.861+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.861+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.861+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.861+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.861+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.866 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.861+0000 7f7c227cb500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:29.866 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.861+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7eaa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.866 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.862+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.866 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.862+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.866 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.862+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.866 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.862+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.866 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.862+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.866 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.862+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.867 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.862+0000 7f7c227cb500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.867 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.862+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7eaa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.870 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.863+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.870 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.863+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.870 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.863+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.870 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.863+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.870 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.863+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.870 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.863+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.870 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.863+0000 7f7c227cb500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:29.870 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.864+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7eaa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.870 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.864+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.864+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.864+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.864+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.864+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.864+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.864+0000 7f7c227cb500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:29.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.865+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7eaa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.865+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.865+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.865+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.871 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.865+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.872 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.865+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.872 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.865+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.872 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.865+0000 7f7c227cb500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.872 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.866+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7eaa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.872 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.866+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.872 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.866+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.872 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.866+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.872 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.866+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.872 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.867+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.872 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.867+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.872 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.867+0000 7f7c227cb500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:29.873 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.867+0000 7f7c227cb500 20 get_system_obj_state: rctx=0x7ffeaec7eaa0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x563dfbe61b60 s->prefetch_data=0 2023-09-22T13:04:29.873 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.868+0000 7f7c227cb500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.873 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.868+0000 7f7c227cb500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.873 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.868+0000 7f7c227cb500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.873 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.868+0000 7f7c227cb500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.873 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.868+0000 7f7c227cb500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.873 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.868+0000 7f7c227cb500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:29.873 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.868+0000 7f7c227cb500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.874 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.870+0000 7f7c227cb500 20 remove_watcher() i=0 2023-09-22T13:04:29.874 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.870+0000 7f7c227cb500 2 removed watcher, disabling cache 2023-09-22T13:04:29.874 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.871+0000 7f7c227cb500 20 remove_watcher() i=1 2023-09-22T13:04:29.874 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.872+0000 7f7c227cb500 20 remove_watcher() i=2 2023-09-22T13:04:29.874 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.873+0000 7f7c227cb500 20 remove_watcher() i=3 2023-09-22T13:04:29.875 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.874+0000 7f7c227cb500 20 remove_watcher() i=4 2023-09-22T13:04:29.875 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.875+0000 7f7c227cb500 20 remove_watcher() i=5 2023-09-22T13:04:29.877 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.876+0000 7f7c227cb500 20 remove_watcher() i=6 2023-09-22T13:04:29.879 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.879+0000 7f7c227cb500 20 remove_watcher() i=7 2023-09-22T13:04:29.880 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.880+0000 7f7bdf7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.885 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: bucket check olh finds leftover OLH entries 2023-09-22T13:04:29.885 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.886 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bucket check olh --bucket check-bucket --dump-keys 2023-09-22T13:04:29.911 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:29.911 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:29.918 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.918+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.920 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.919+0000 7f67c56dc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.920 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.919+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.920+0000 7f67c56dc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.920+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.920+0000 7f67c56dc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.920+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.921 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.921+0000 7f67c56dc500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.931 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.930+0000 7f67c56dc500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:29.931 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.930+0000 7f67c56dc500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:29.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.930+0000 7f67c56dc500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:29.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.930+0000 7f67c56dc500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:29.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.930+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.930+0000 7f67c56dc500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.930+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.931+0000 7f67c56dc500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.931+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.931+0000 7f67c56dc500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.932 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.931+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.935 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.932+0000 7f67c56dc500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.935 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.932+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.935 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.932+0000 7f67c56dc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.935 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.932+0000 7f67c56dc500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:29.935 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.932+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.935 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.932+0000 7f67c56dc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.935 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.932+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.935 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.932+0000 7f67c56dc500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.935 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.932+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 zonegroup default 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 10 Cannot find current period zone using local zone 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:29.936 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 zone default found 2023-09-22T13:04:29.937 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.933+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.937 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.934+0000 7f67c56dc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.937 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.934+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.937 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.934+0000 7f67c56dc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:29.937 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.934+0000 7f67c56dc500 20 started sync module instance, tier type = 2023-09-22T13:04:29.937 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.934+0000 7f67c56dc500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:29.956 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.955+0000 7f67c56dc500 20 add_watcher() i=0 2023-09-22T13:04:29.956 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.955+0000 7f67c56dc500 20 add_watcher() i=1 2023-09-22T13:04:29.956 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.955+0000 7f67c56dc500 20 add_watcher() i=2 2023-09-22T13:04:29.956 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.955+0000 7f67c56dc500 20 add_watcher() i=3 2023-09-22T13:04:29.956 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.955+0000 7f67c56dc500 20 add_watcher() i=4 2023-09-22T13:04:29.956 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.955+0000 7f67c56dc500 20 add_watcher() i=5 2023-09-22T13:04:29.958 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.957+0000 7f67abfff700 0 -- 172.21.15.170:0/2137052902 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f67a4029370 msgr2=0x7f67a40297b0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:29.958 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.958+0000 7f67c56dc500 20 add_watcher() i=6 2023-09-22T13:04:29.964 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.964+0000 7f67c56dc500 20 add_watcher() i=7 2023-09-22T13:04:29.964 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.964+0000 7f67c56dc500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:29.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.965+0000 7f6781fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.966+0000 7f67c56dc500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:29.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.966+0000 7f67c56dc500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:29.970 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.970+0000 7f67abfff700 0 -- 172.21.15.170:0/2590465332 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x55bc4d77d060 msgr2=0x55bc4d78ef90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:29.972 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.972+0000 7f67c56dc500 5 note: GC not initialized 2023-09-22T13:04:29.974 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f66f8ff9700 20 reqs_thread_entry: start 2023-09-22T13:04:29.974 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f66ee7fc700 20 reqs_thread_entry: start 2023-09-22T13:04:29.974 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f67c56dc500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:29.974 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f67c56dc500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:29.974 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f66ecff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:00 2023 2023-09-22T13:04:29.974 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.975 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f66d3fff700 20 reqs_thread_entry: start 2023-09-22T13:04:29.975 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f67c56dc500 20 get_system_obj_state: rctx=0x7ffe224d1cc8 obj=default.rgw.meta:root:check-bucket state=0x55bc4d6e2b80 s->prefetch_data=0 2023-09-22T13:04:29.975 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f67c56dc500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:04:29.975 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f67c56dc500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:04:29.975 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f67c56dc500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:29.975 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f67c56dc500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.975 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f67c56dc500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.975 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f67c56dc500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:04:29.975 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.973+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.977 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:29.977 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:04:29.977 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:29.977 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 20 get_system_obj_state: rctx=0x7ffe224d1d98 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55bc4d6e2b80 s->prefetch_data=0 2023-09-22T13:04:29.977 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : miss 2023-09-22T13:04:29.977 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x16 2023-09-22T13:04:29.977 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:29.977 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.977 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:29.978 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:29.978 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:29.978 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:29.978 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:04:29.978 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.974+0000 7f67c56dc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:29.978 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.975+0000 7f67c56dc500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:29.978 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.975+0000 7f67c56dc500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x13 2023-09-22T13:04:29.978 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.975+0000 7f67c56dc500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:29.978 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.975+0000 7f67c56dc500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:29.978 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.975+0000 7f67c56dc500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:29.978 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.975+0000 7f67c56dc500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 2023-09-22T13:04:29.979 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.975+0000 7f67c56dc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:29.979 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.975+0000 7f67c56dc500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:04:29.979 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.975+0000 7f67c56dc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:29.979 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.976+0000 7f67c56dc500 1 NOTICE: finished shard 1 (0 entries found) 2023-09-22T13:04:29.979 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.976+0000 7f67c56dc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:29.979 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.976+0000 7f67c56dc500 1 NOTICE: finished shard 2 (0 entries found) 2023-09-22T13:04:29.979 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.976+0000 7f67c56dc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:29.979 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.976+0000 7f67c56dc500 1 NOTICE: finished shard 3 (0 entries found) 2023-09-22T13:04:29.979 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.976+0000 7f67c56dc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:29.980 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.977+0000 7f67c56dc500 1 NOTICE: finished shard 4 (0 entries found) 2023-09-22T13:04:29.981 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.977+0000 7f67c56dc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:29.981 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.977+0000 7f67c56dc500 1 NOTICE: finished shard 5 (1 entries found) 2023-09-22T13:04:29.981 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.977+0000 7f67c56dc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:29.981 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.978+0000 7f67c56dc500 1 NOTICE: finished shard 6 (0 entries found) 2023-09-22T13:04:29.981 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.978+0000 7f67c56dc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:29.981 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.978+0000 7f67c56dc500 1 NOTICE: finished shard 7 (0 entries found) 2023-09-22T13:04:29.981 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.978+0000 7f67c56dc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:29.981 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.979+0000 7f67c56dc500 1 NOTICE: finished shard 8 (1 entries found) 2023-09-22T13:04:29.981 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.979+0000 7f67c56dc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:29.981 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.979+0000 7f67c56dc500 1 NOTICE: finished shard 9 (0 entries found) 2023-09-22T13:04:29.981 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.979+0000 7f67c56dc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:29.982 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.979+0000 7f67c56dc500 1 NOTICE: finished shard 10 (0 entries found) 2023-09-22T13:04:29.982 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.980+0000 7f67c56dc500 1 NOTICE: finished all shards (2 entries found) 2023-09-22T13:04:29.982 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.981+0000 7f67c56dc500 20 remove_watcher() i=0 2023-09-22T13:04:29.982 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.981+0000 7f67c56dc500 2 removed watcher, disabling cache 2023-09-22T13:04:29.982 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.982+0000 7f67c56dc500 20 remove_watcher() i=1 2023-09-22T13:04:29.983 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.983+0000 7f67c56dc500 20 remove_watcher() i=2 2023-09-22T13:04:29.984 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.984+0000 7f67c56dc500 20 remove_watcher() i=3 2023-09-22T13:04:29.985 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.985+0000 7f67c56dc500 20 remove_watcher() i=4 2023-09-22T13:04:29.986 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.986+0000 7f67c56dc500 20 remove_watcher() i=5 2023-09-22T13:04:29.987 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.987+0000 7f67c56dc500 20 remove_watcher() i=6 2023-09-22T13:04:29.988 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.988+0000 7f67c56dc500 20 remove_watcher() i=7 2023-09-22T13:04:29.989 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:29.989+0000 7f6781fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:29.994 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: bucket check olh fixes leftover OLH entries 2023-09-22T13:04:29.994 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:29.995 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bucket check olh --bucket check-bucket --fix --rgw-olh-pending-timeout-sec 0 --dump-keys 2023-09-22T13:04:30.022 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:30.022 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:30.030 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.030+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.032 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.031+0000 7f7b78e4d500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.032 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.031+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.033 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.032+0000 7f7b78e4d500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.033 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.032+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.033 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.032+0000 7f7b78e4d500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.033 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.032+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.033 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.033+0000 7f7b78e4d500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.044 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.043+0000 7f7b78e4d500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:30.044 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.043+0000 7f7b78e4d500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:30.044 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.043+0000 7f7b78e4d500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:30.044 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.043+0000 7f7b78e4d500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:30.044 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.043+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.044 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.043+0000 7f7b78e4d500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.044 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.043+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.044 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.043+0000 7f7b78e4d500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:30.044 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.043+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.045 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.043+0000 7f7b78e4d500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.045 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.043+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.046 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.044+0000 7f7b78e4d500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:30.046 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.044+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.046 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.044+0000 7f7b78e4d500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.046 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.044+0000 7f7b78e4d500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:30.046 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.044+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.046 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.045+0000 7f7b78e4d500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.047 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.045+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.047 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.045+0000 7f7b78e4d500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.047 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.045+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.047 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.045+0000 7f7b78e4d500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:30.047 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.045+0000 7f7b78e4d500 20 zonegroup default 2023-09-22T13:04:30.047 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.045+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.047 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.045+0000 7f7b78e4d500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.047 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.045+0000 7f7b78e4d500 10 Cannot find current period zone using local zone 2023-09-22T13:04:30.047 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.045+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.047 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.046+0000 7f7b78e4d500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.047 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.046+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.048 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.046+0000 7f7b78e4d500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.048 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.046+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.046+0000 7f7b78e4d500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:30.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.046+0000 7f7b78e4d500 20 zone default found 2023-09-22T13:04:30.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.046+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.046+0000 7f7b78e4d500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.046+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.047+0000 7f7b78e4d500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.047+0000 7f7b78e4d500 20 started sync module instance, tier type = 2023-09-22T13:04:30.049 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.047+0000 7f7b78e4d500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:30.054 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.053+0000 7f7b63b2e700 0 -- 172.21.15.170:0/3823429732 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f7b5804dec0 msgr2=0x7f7b5801f640 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:30.057 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.057+0000 7f7b78e4d500 20 add_watcher() i=0 2023-09-22T13:04:30.057 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.057+0000 7f7b78e4d500 20 add_watcher() i=1 2023-09-22T13:04:30.057 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.057+0000 7f7b78e4d500 20 add_watcher() i=2 2023-09-22T13:04:30.058 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.057+0000 7f7b78e4d500 20 add_watcher() i=3 2023-09-22T13:04:30.058 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.057+0000 7f7b78e4d500 20 add_watcher() i=4 2023-09-22T13:04:30.058 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.057+0000 7f7b78e4d500 20 add_watcher() i=5 2023-09-22T13:04:30.058 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.057+0000 7f7b78e4d500 20 add_watcher() i=6 2023-09-22T13:04:30.058 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.058+0000 7f7b78e4d500 20 add_watcher() i=7 2023-09-22T13:04:30.058 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.058+0000 7f7b78e4d500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:30.060 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.059+0000 7f7b35fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:30.060 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.060+0000 7f7b78e4d500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:30.060 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.060+0000 7f7b78e4d500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:30.064 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.064+0000 7f7b63b2e700 0 -- 172.21.15.170:0/4105466215 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x55b4a8970810 msgr2=0x55b4a896f890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:30.066 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.066+0000 7f7b78e4d500 5 note: GC not initialized 2023-09-22T13:04:30.068 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.066+0000 7f7aacff9700 20 reqs_thread_entry: start 2023-09-22T13:04:30.068 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7aa27fc700 20 reqs_thread_entry: start 2023-09-22T13:04:30.068 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7b78e4d500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:30.068 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7b78e4d500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:30.069 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7aa0ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:00 2023 2023-09-22T13:04:30.069 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:30.069 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7a87fff700 20 reqs_thread_entry: start 2023-09-22T13:04:30.069 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7b78e4d500 20 get_system_obj_state: rctx=0x7ffc6bc0c298 obj=default.rgw.meta:root:check-bucket state=0x55b4a87b1810 s->prefetch_data=0 2023-09-22T13:04:30.069 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7b78e4d500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:04:30.069 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7b78e4d500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:04:30.069 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7b78e4d500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:30.069 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7b78e4d500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:30.069 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7b78e4d500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.069 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7b78e4d500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:04:30.069 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.067+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.071 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:30.071 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:04:30.071 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:30.071 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 20 get_system_obj_state: rctx=0x7ffc6bc0c368 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b4a87b1810 s->prefetch_data=0 2023-09-22T13:04:30.071 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : miss 2023-09-22T13:04:30.071 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x16 2023-09-22T13:04:30.071 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:30.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:30.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:30.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:04:30.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.068+0000 7f7b78e4d500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.069+0000 7f7b78e4d500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.069+0000 7f7b78e4d500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x13 2023-09-22T13:04:30.072 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.069+0000 7f7b78e4d500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:30.073 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.069+0000 7f7b78e4d500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:30.073 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.069+0000 7f7b78e4d500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:30.073 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.069+0000 7f7b78e4d500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 2023-09-22T13:04:30.073 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.069+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:30.073 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.069+0000 7f7b78e4d500 1 NOTICE: finished shard 0 (0 entries removed) 2023-09-22T13:04:30.073 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.069+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:30.073 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.070+0000 7f7b78e4d500 1 NOTICE: finished shard 1 (0 entries removed) 2023-09-22T13:04:30.073 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.070+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:30.073 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.070+0000 7f7b78e4d500 1 NOTICE: finished shard 2 (0 entries removed) 2023-09-22T13:04:30.073 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.070+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:30.075 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.071+0000 7f7b78e4d500 1 NOTICE: finished shard 3 (0 entries removed) 2023-09-22T13:04:30.075 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.071+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:30.075 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.072+0000 7f7b78e4d500 1 NOTICE: finished shard 4 (0 entries removed) 2023-09-22T13:04:30.075 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.072+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:30.075 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.072+0000 7f7b78e4d500 20 get_obj_state: rctx=0x55b4a8a04980 obj=check-bucket:d state=0x55b4a8a07d68 s->prefetch_data=0 2023-09-22T13:04:30.075 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.072+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:30.075 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.072+0000 7f7b78e4d500 1 NOTICE: finished shard 6 (0 entries removed) 2023-09-22T13:04:30.076 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.072+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:30.076 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.073+0000 7f7b78e4d500 1 NOTICE: finished shard 7 (0 entries removed) 2023-09-22T13:04:30.076 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.073+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:30.076 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.074+0000 7f7b78e4d500 20 get_obj_state: rctx=0x55b4a8a28480 obj=check-bucket:c state=0x55b4a8a28ec8 s->prefetch_data=0 2023-09-22T13:04:30.076 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.074+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:30.076 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.074+0000 7f7b78e4d500 1 NOTICE: finished shard 9 (0 entries removed) 2023-09-22T13:04:30.076 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.074+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:30.076 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.074+0000 7f7b78e4d500 1 NOTICE: finished shard 10 (0 entries removed) 2023-09-22T13:04:30.076 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.074+0000 7f7b78e4d500 20 get_obj_state: setting s->obj_tag to qj18wi4cgp7z2ph3fwk2w8peez1vmhfc 2023-09-22T13:04:30.076 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.074+0000 7f7b78e4d500 20 get_obj_state_impl: setting s->olh_tag to 1ormwvc1x16rnao5v4juvavrjcc1xaiu 2023-09-22T13:04:30.076 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.074+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:30.077 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.074+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:30.077 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.075+0000 7f7b78e4d500 20 olh_log_entry: epoch=4 op=2 key=d[] 2023-09-22T13:04:30.077 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.075+0000 7f7b78e4d500 20 olh_log_entry: epoch=4 op=3 key=d[3o.jCjA.6SNImo9HldyooQF5eZ3Iv7M] 2023-09-22T13:04:30.077 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.075+0000 7f7b78e4d500 20 get_obj_state: rctx=0x55b4a8a04980 obj=check-bucket:_:3o.jCjA.6SNImo9HldyooQF5eZ3Iv7M_d state=0x55b4a8a30ee8 s->prefetch_data=0 2023-09-22T13:04:30.078 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.077+0000 7f7b78e4d500 20 get_obj_state: rctx=0x55b4a8a04980 obj=check-bucket:d state=0x55b4a8a07d68 s->prefetch_data=0 2023-09-22T13:04:30.078 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.077+0000 7f7b78e4d500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=1ormwvc1x16rnao5v4juvavrjcc1xaiu 2023-09-22T13:04:30.080 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.079+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:30.082 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.081+0000 7f7b78e4d500 1 NOTICE: finished shard 5 (1 entries removed) 2023-09-22T13:04:30.082 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.081+0000 7f7b78e4d500 20 get_obj_state: setting s->obj_tag to d2l5h49vf0eahw30o1omgem89lcc7dwq 2023-09-22T13:04:30.082 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.081+0000 7f7b78e4d500 20 get_obj_state_impl: setting s->olh_tag to 7q00u7gs97dsxej1yly67972ei7mssbp 2023-09-22T13:04:30.082 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.081+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:30.082 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.081+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:30.082 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.081+0000 7f7b78e4d500 20 olh_log_entry: epoch=4 op=2 key=c[] 2023-09-22T13:04:30.082 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.081+0000 7f7b78e4d500 20 olh_log_entry: epoch=4 op=3 key=c[OLokwOkwKWOz85wp9fzahuOD3DL7c.o] 2023-09-22T13:04:30.082 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.081+0000 7f7b78e4d500 20 get_obj_state: rctx=0x55b4a8a28480 obj=check-bucket:_:OLokwOkwKWOz85wp9fzahuOD3DL7c.o_c state=0x55b4a8a07d68 s->prefetch_data=0 2023-09-22T13:04:30.083 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.082+0000 7f7b78e4d500 20 get_obj_state: rctx=0x55b4a8a28480 obj=check-bucket:c state=0x55b4a8a28ec8 s->prefetch_data=0 2023-09-22T13:04:30.083 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.082+0000 7f7b78e4d500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=7q00u7gs97dsxej1yly67972ei7mssbp 2023-09-22T13:04:30.086 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.085+0000 7f7b78e4d500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:30.087 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.086+0000 7f7b78e4d500 1 NOTICE: finished shard 8 (1 entries removed) 2023-09-22T13:04:30.087 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.086+0000 7f7b78e4d500 1 NOTICE: finished all shards (2 entries removed) 2023-09-22T13:04:30.088 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.087+0000 7f7b78e4d500 20 remove_watcher() i=0 2023-09-22T13:04:30.088 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.087+0000 7f7b78e4d500 2 removed watcher, disabling cache 2023-09-22T13:04:30.089 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.089+0000 7f7b78e4d500 20 remove_watcher() i=1 2023-09-22T13:04:30.090 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.090+0000 7f7b78e4d500 20 remove_watcher() i=2 2023-09-22T13:04:30.091 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.091+0000 7f7b78e4d500 20 remove_watcher() i=3 2023-09-22T13:04:30.092 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.092+0000 7f7b78e4d500 20 remove_watcher() i=4 2023-09-22T13:04:30.093 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.093+0000 7f7b78e4d500 20 remove_watcher() i=5 2023-09-22T13:04:30.094 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.094+0000 7f7b78e4d500 20 remove_watcher() i=6 2023-09-22T13:04:30.096 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.095+0000 7f7b78e4d500 20 remove_watcher() i=7 2023-09-22T13:04:30.096 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.096+0000 7f7b35fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:30.102 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:04:30.126 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:30.127 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:30.134 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.134+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.136 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.135+0000 7f2a5e0fd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.136 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.135+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.137 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.136+0000 7f2a5e0fd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.137 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.136+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.137 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.136+0000 7f2a5e0fd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.137 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.136+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.137 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.137+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.147 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.146+0000 7f2a5e0fd500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:30.147 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.146+0000 7f2a5e0fd500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:30.147 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.146+0000 7f2a5e0fd500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:30.147 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.146+0000 7f2a5e0fd500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:30.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.146+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.146+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.146+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.146+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:30.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.146+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.147+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.148 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.147+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.147+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:30.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.147+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.147+0000 7f2a5e0fd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.147+0000 7f2a5e0fd500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:30.150 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.147+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.148+0000 7f2a5e0fd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.148+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.148+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.148+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.148+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.148+0000 7f2a5e0fd500 20 zonegroup default 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.148+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.148+0000 7f2a5e0fd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.148+0000 7f2a5e0fd500 10 Cannot find current period zone using local zone 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.148+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.149+0000 7f2a5e0fd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.151 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.149+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.152 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.149+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.152 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.149+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.152 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.149+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:30.152 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.149+0000 7f2a5e0fd500 20 zone default found 2023-09-22T13:04:30.152 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.149+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.152 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.149+0000 7f2a5e0fd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.152 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.149+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.152 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.150+0000 7f2a5e0fd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.152 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.150+0000 7f2a5e0fd500 20 started sync module instance, tier type = 2023-09-22T13:04:30.152 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.150+0000 7f2a5e0fd500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:30.163 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.162+0000 7f2a5e0fd500 20 add_watcher() i=0 2023-09-22T13:04:30.163 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.162+0000 7f2a5e0fd500 20 add_watcher() i=1 2023-09-22T13:04:30.163 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.162+0000 7f2a5e0fd500 20 add_watcher() i=2 2023-09-22T13:04:30.163 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.162+0000 7f2a5e0fd500 20 add_watcher() i=3 2023-09-22T13:04:30.163 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.162+0000 7f2a5e0fd500 20 add_watcher() i=4 2023-09-22T13:04:30.163 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.162+0000 7f2a5e0fd500 20 add_watcher() i=5 2023-09-22T13:04:30.166 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.165+0000 7f2a5e0fd500 20 add_watcher() i=6 2023-09-22T13:04:30.169 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.168+0000 7f2a5e0fd500 20 add_watcher() i=7 2023-09-22T13:04:30.169 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.168+0000 7f2a5e0fd500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:30.169 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.169+0000 7f2a49de0700 0 -- 172.21.15.170:0/1346226421 >> [v2:172.21.15.191:6816/144992,v1:172.21.15.191:6817/144992] conn(0x7f2a40025930 msgr2=0x7f2a40023400 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:30.171 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.171+0000 7f2a1a7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:30.172 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.171+0000 7f2a5e0fd500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:30.172 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.171+0000 7f2a5e0fd500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:30.176 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.175+0000 7f2a495df700 0 -- 172.21.15.170:0/2807993859 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x5569d4cc7fb0 msgr2=0x5569d4cc0f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:30.178 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.177+0000 7f2a5e0fd500 5 note: GC not initialized 2023-09-22T13:04:30.179 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.178+0000 7f29897fa700 20 reqs_thread_entry: start 2023-09-22T13:04:30.179 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.178+0000 7f2982ffd700 20 reqs_thread_entry: start 2023-09-22T13:04:30.179 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.178+0000 7f2a5e0fd500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:30.179 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.178+0000 7f2a5e0fd500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:30.179 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.178+0000 7f29817fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:00 2023 2023-09-22T13:04:30.180 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:30.180 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.179+0000 7f2980ff9700 20 reqs_thread_entry: start 2023-09-22T13:04:30.180 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.179+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff63330e0 obj=default.rgw.meta:root:check-bucket state=0x5569d4ccf180 s->prefetch_data=0 2023-09-22T13:04:30.182 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.179+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.182 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.179+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.182 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.179+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:30.182 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.179+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff63330e0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.182 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.180+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.182 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.180+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.182 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.180+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.180+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.180+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.180+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.180+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff6332eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.180+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.180+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.180+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.180+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.181+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.181+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.181+0000 7f2a5e0fd500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:30.183 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.181+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff6332eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.184 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.181+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.184 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.181+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.184 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.181+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.184 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.181+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.187 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.182+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.182+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.182+0000 7f2a5e0fd500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:30.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.182+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff6332eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.183+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.183+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.183+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.183+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.183+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.183+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.183+0000 7f2a5e0fd500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:30.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.183+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff6332eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.184+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.184+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.184+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.184+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.184+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.184+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.184+0000 7f2a5e0fd500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.184+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff6332eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.185+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.185+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.185+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.185+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.185+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.185+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.185+0000 7f2a5e0fd500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.185+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff6332eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.186+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.186+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.186+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.186+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.186+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.186+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.186+0000 7f2a5e0fd500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.186+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff6332eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.190 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.187+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.191 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.187+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.191 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.187+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.191 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.187+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.191 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.187+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.191 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.187+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.191 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.187+0000 7f2a5e0fd500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:30.193 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.187+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff6332eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.194 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.188+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.194 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.188+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.194 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.188+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.194 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.188+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.194 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.188+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.194 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.188+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.194 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.188+0000 7f2a5e0fd500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:30.194 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.188+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff6332eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.194 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.189+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.194 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.189+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.189+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.189+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.189+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.189+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.189+0000 7f2a5e0fd500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.190+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff6332eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.190+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.190+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.190+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.190+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.190+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.195 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.190+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.196 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.190+0000 7f2a5e0fd500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:30.196 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.191+0000 7f2a5e0fd500 20 get_system_obj_state: rctx=0x7ffff6332eb0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5569d4bc1b60 s->prefetch_data=0 2023-09-22T13:04:30.196 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.191+0000 7f2a5e0fd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.196 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.191+0000 7f2a5e0fd500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.196 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.191+0000 7f2a5e0fd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.196 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.191+0000 7f2a5e0fd500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.196 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.192+0000 7f2a5e0fd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.196 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.192+0000 7f2a5e0fd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.196 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.192+0000 7f2a5e0fd500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:30.197 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.194+0000 7f2a5e0fd500 20 remove_watcher() i=0 2023-09-22T13:04:30.197 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.194+0000 7f2a5e0fd500 2 removed watcher, disabling cache 2023-09-22T13:04:30.197 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.195+0000 7f2a5e0fd500 20 remove_watcher() i=1 2023-09-22T13:04:30.197 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.196+0000 7f2a5e0fd500 20 remove_watcher() i=2 2023-09-22T13:04:30.197 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.197+0000 7f2a5e0fd500 20 remove_watcher() i=3 2023-09-22T13:04:30.198 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.198+0000 7f2a5e0fd500 20 remove_watcher() i=4 2023-09-22T13:04:30.200 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.199+0000 7f2a5e0fd500 20 remove_watcher() i=5 2023-09-22T13:04:30.201 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.200+0000 7f2a5e0fd500 20 remove_watcher() i=6 2023-09-22T13:04:30.203 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.202+0000 7f2a5e0fd500 20 remove_watcher() i=7 2023-09-22T13:04:30.203 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.203+0000 7f2a1a7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:30.210 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:04:30.234 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:30.234 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:30.242 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.241+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.244 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.243+0000 7f2c9ca65500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.244 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.243+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.244 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.244+0000 7f2c9ca65500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.244 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.244+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.245 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.244+0000 7f2c9ca65500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.245 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.244+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.245 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.245+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.254 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.254+0000 7f2c9ca65500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:30.255 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.254+0000 7f2c9ca65500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:30.255 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.254+0000 7f2c9ca65500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:30.255 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.254+0000 7f2c9ca65500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:30.255 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.254+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.255 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.254+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.255 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.254+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.256 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.254+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:30.256 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.254+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.256 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.255+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.256 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.255+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.255+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:30.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.255+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.255+0000 7f2c9ca65500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.255+0000 7f2c9ca65500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:30.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.255+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.256+0000 7f2c9ca65500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.256+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.256+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.256+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.256+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:30.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.256+0000 7f2c9ca65500 20 zonegroup default 2023-09-22T13:04:30.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.256+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.256+0000 7f2c9ca65500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.256+0000 7f2c9ca65500 10 Cannot find current period zone using local zone 2023-09-22T13:04:30.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.256+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.257+0000 7f2c9ca65500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.257+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.257+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.257+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.257+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:30.260 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.257+0000 7f2c9ca65500 20 zone default found 2023-09-22T13:04:30.260 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.257+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.260 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.257+0000 7f2c9ca65500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.260 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.257+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.260 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.258+0000 7f2c9ca65500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.260 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.258+0000 7f2c9ca65500 20 started sync module instance, tier type = 2023-09-22T13:04:30.260 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.258+0000 7f2c9ca65500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:30.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.276+0000 7f2c9ca65500 20 add_watcher() i=0 2023-09-22T13:04:30.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.276+0000 7f2c9ca65500 20 add_watcher() i=1 2023-09-22T13:04:30.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.276+0000 7f2c9ca65500 20 add_watcher() i=2 2023-09-22T13:04:30.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.276+0000 7f2c9ca65500 20 add_watcher() i=3 2023-09-22T13:04:30.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.276+0000 7f2c9ca65500 20 add_watcher() i=4 2023-09-22T13:04:30.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.276+0000 7f2c9ca65500 20 add_watcher() i=5 2023-09-22T13:04:30.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.276+0000 7f2c9ca65500 20 add_watcher() i=6 2023-09-22T13:04:30.277 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.276+0000 7f2c9ca65500 20 add_watcher() i=7 2023-09-22T13:04:30.277 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.276+0000 7f2c9ca65500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:30.277 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.277+0000 7f2c88748700 0 -- 172.21.15.170:0/2320618825 >> [v2:172.21.15.191:6816/144992,v1:172.21.15.191:6817/144992] conn(0x7f2c7c02a400 msgr2=0x7f2c7c02a840 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:30.279 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.279+0000 7f2c597e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:30.279 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.279+0000 7f2c9ca65500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:30.280 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.279+0000 7f2c9ca65500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:30.284 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.283+0000 7f2c87f47700 0 -- 172.21.15.170:0/1970533821 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x55b0148f3b40 msgr2=0x55b0148eddb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:30.484 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.484+0000 7f2c87f47700 0 --2- 172.21.15.170:0/1970533821 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x55b0148f3b40 0x55b0148eddb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:04:30.487 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.486+0000 7f2c9ca65500 5 note: GC not initialized 2023-09-22T13:04:30.488 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.487+0000 7f2bc7fff700 20 reqs_thread_entry: start 2023-09-22T13:04:30.488 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.487+0000 7f2bc5ffb700 20 reqs_thread_entry: start 2023-09-22T13:04:30.488 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.487+0000 7f2c9ca65500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:30.489 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.487+0000 7f2c9ca65500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:30.489 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.487+0000 7f2bb3fff700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:00 2023 2023-09-22T13:04:30.489 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:30.489 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.488+0000 7f2bb37fe700 20 reqs_thread_entry: start 2023-09-22T13:04:30.489 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.488+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017768d0 obj=default.rgw.meta:root:check-bucket state=0x55b0148ee520 s->prefetch_data=0 2023-09-22T13:04:30.491 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.488+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.491 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.488+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.491 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.488+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:30.491 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.488+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017768d0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.491 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.489+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.491 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.489+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.492 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.489+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.492 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.489+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.492 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.489+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.492 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.489+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.492 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.489+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017766a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.492 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.489+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.492 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.489+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.492 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.489+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.493 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.489+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.493 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.490+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.493 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.490+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.493 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.490+0000 7f2c9ca65500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:30.493 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.490+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017766a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.493 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.490+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.493 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.490+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.493 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.490+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.493 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.490+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.496 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.491+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.496 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.491+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.496 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.491+0000 7f2c9ca65500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:30.496 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.491+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017766a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.496 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.492+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.496 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.492+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.496 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.492+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.496 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.492+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.496 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.492+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.496 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.492+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.492+0000 7f2c9ca65500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.492+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017766a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.493+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.493+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.493+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.493+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.493+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.493+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.493+0000 7f2c9ca65500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.494+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017766a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.494+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.497 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.494+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.494+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.494+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.494+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.494+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.494+0000 7f2c9ca65500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.495+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017766a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.495+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.495+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.495+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.495+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.495+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.498 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.495+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.499 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.495+0000 7f2c9ca65500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:30.501 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.496+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017766a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.501 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.496+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.501 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.496+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.501 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.496+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.501 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.496+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.501 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.496+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.501 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.496+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.501 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.496+0000 7f2c9ca65500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:30.501 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.497+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017766a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.501 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.497+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.497+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.497+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.497+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.497+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.497+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.497+0000 7f2c9ca65500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.498+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017766a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.498+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.498+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.498+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.498+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.502 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.498+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.503 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.498+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.503 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.498+0000 7f2c9ca65500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:30.503 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.499+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017766a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.503 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.499+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.503 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.499+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.503 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.499+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.503 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.499+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.503 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.499+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.503 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.499+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.503 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.499+0000 7f2c9ca65500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:30.503 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.500+0000 7f2c9ca65500 20 get_system_obj_state: rctx=0x7ffd017766a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55b0147eeb60 s->prefetch_data=0 2023-09-22T13:04:30.504 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.500+0000 7f2c9ca65500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.504 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.500+0000 7f2c9ca65500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.504 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.500+0000 7f2c9ca65500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.504 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.500+0000 7f2c9ca65500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.505 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.501+0000 7f2c9ca65500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.505 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.501+0000 7f2c9ca65500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:30.505 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.501+0000 7f2c9ca65500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:30.505 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.503+0000 7f2c9ca65500 20 remove_watcher() i=0 2023-09-22T13:04:30.505 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.503+0000 7f2c9ca65500 2 removed watcher, disabling cache 2023-09-22T13:04:30.505 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.504+0000 7f2c9ca65500 20 remove_watcher() i=1 2023-09-22T13:04:30.505 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.505+0000 7f2c9ca65500 20 remove_watcher() i=2 2023-09-22T13:04:30.506 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.506+0000 7f2c9ca65500 20 remove_watcher() i=3 2023-09-22T13:04:30.508 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.507+0000 7f2c9ca65500 20 remove_watcher() i=4 2023-09-22T13:04:30.509 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.508+0000 7f2c9ca65500 20 remove_watcher() i=5 2023-09-22T13:04:30.510 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.509+0000 7f2c9ca65500 20 remove_watcher() i=6 2023-09-22T13:04:30.511 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.511+0000 7f2c9ca65500 20 remove_watcher() i=7 2023-09-22T13:04:30.512 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.511+0000 7f2c597e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:30.516 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: bucket check olh does not find new leftover OLH entries after fix 2023-09-22T13:04:30.517 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:30.518 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bucket check olh --bucket check-bucket --dump-keys 2023-09-22T13:04:30.543 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:30.543 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:30.551 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.551+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.553 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.552+0000 7f8565740500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.553 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.552+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.554 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.553+0000 7f8565740500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.554 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.553+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.554 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.553+0000 7f8565740500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.554 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.553+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.554 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.554+0000 7f8565740500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.564 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.564+0000 7f8565740500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:30.564 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.564+0000 7f8565740500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:30.564 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.564+0000 7f8565740500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:30.565 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.564+0000 7f8565740500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:30.565 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.564+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.565 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.564+0000 7f8565740500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.565 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.564+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.566 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.564+0000 7f8565740500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:30.566 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.564+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.566 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.564+0000 7f8565740500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.566 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.564+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.566 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.565+0000 7f8565740500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:30.566 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.565+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.566 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.565+0000 7f8565740500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.567 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.565+0000 7f8565740500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:30.567 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.565+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.567 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.565+0000 7f8565740500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.567 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.565+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.567 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.565+0000 7f8565740500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.567 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.565+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.568 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.566+0000 7f8565740500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:30.568 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.566+0000 7f8565740500 20 zonegroup default 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.566+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.566+0000 7f8565740500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.566+0000 7f8565740500 10 Cannot find current period zone using local zone 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.566+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.566+0000 7f8565740500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.566+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.567+0000 7f8565740500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.567+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.567+0000 7f8565740500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.567+0000 7f8565740500 20 zone default found 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.567+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.567+0000 7f8565740500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.569 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.567+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.570 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.568+0000 7f8565740500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:30.570 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.568+0000 7f8565740500 20 started sync module instance, tier type = 2023-09-22T13:04:30.570 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.568+0000 7f8565740500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:30.578 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.577+0000 7f8565740500 20 add_watcher() i=0 2023-09-22T13:04:30.578 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.577+0000 7f8565740500 20 add_watcher() i=1 2023-09-22T13:04:30.578 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.577+0000 7f8565740500 20 add_watcher() i=2 2023-09-22T13:04:30.578 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.577+0000 7f8565740500 20 add_watcher() i=3 2023-09-22T13:04:30.578 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.577+0000 7f8565740500 20 add_watcher() i=4 2023-09-22T13:04:30.578 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.577+0000 7f8565740500 20 add_watcher() i=5 2023-09-22T13:04:30.579 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.577+0000 7f8565740500 20 add_watcher() i=6 2023-09-22T13:04:30.579 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.578+0000 7f854bfff700 0 -- 172.21.15.170:0/826914014 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f854402b3f0 msgr2=0x7f854402b830 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:30.579 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.578+0000 7f8565740500 20 add_watcher() i=7 2023-09-22T13:04:30.579 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.578+0000 7f8565740500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:30.581 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.580+0000 7f8521fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:30.581 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.580+0000 7f8565740500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:30.581 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.580+0000 7f8565740500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:30.584 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.584+0000 7f854bfff700 0 -- 172.21.15.170:0/1276102536 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x55d80350de90 msgr2=0x55d8034fe670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:30.586 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.586+0000 7f8565740500 5 note: GC not initialized 2023-09-22T13:04:30.588 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.587+0000 7f8498ff9700 20 reqs_thread_entry: start 2023-09-22T13:04:30.588 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.587+0000 7f848e7fc700 20 reqs_thread_entry: start 2023-09-22T13:04:30.588 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.587+0000 7f8565740500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:30.588 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.587+0000 7f8565740500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:30.588 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.587+0000 7f848cff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:01 2023 2023-09-22T13:04:30.589 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:30.589 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.587+0000 7f8473fff700 20 reqs_thread_entry: start 2023-09-22T13:04:30.589 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 20 get_system_obj_state: rctx=0x7ffc0cb316a8 obj=default.rgw.meta:root:check-bucket state=0x55d8034a9ab0 s->prefetch_data=0 2023-09-22T13:04:30.589 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:04:30.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:04:30.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:30.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:30.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:04:30.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:30.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:04:30.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:30.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 20 get_system_obj_state: rctx=0x7ffc0cb31778 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55d8034a9ab0 s->prefetch_data=0 2023-09-22T13:04:30.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.588+0000 7f8565740500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : miss 2023-09-22T13:04:30.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x16 2023-09-22T13:04:30.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:30.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:30.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:30.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:30.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:30.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:30.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:04:30.593 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 20 rados->read ofs=0 len=0 2023-09-22T13:04:30.593 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:30.593 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x13 2023-09-22T13:04:30.593 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:30.593 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:30.593 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:30.593 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 2023-09-22T13:04:30.593 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.589+0000 7f8565740500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:30.593 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.590+0000 7f8565740500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:04:30.593 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.590+0000 7f8565740500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:30.593 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.590+0000 7f8565740500 1 NOTICE: finished shard 1 (0 entries found) 2023-09-22T13:04:30.594 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.590+0000 7f8565740500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:30.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.591+0000 7f8565740500 1 NOTICE: finished shard 2 (0 entries found) 2023-09-22T13:04:30.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.591+0000 7f8565740500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:30.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.591+0000 7f8565740500 1 NOTICE: finished shard 3 (0 entries found) 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.591+0000 7f8565740500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.591+0000 7f8565740500 1 NOTICE: finished shard 4 (0 entries found) 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.591+0000 7f8565740500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.592+0000 7f8565740500 1 NOTICE: finished shard 5 (0 entries found) 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.592+0000 7f8565740500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.592+0000 7f8565740500 1 NOTICE: finished shard 6 (0 entries found) 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.592+0000 7f8565740500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.593+0000 7f8565740500 1 NOTICE: finished shard 7 (0 entries found) 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.593+0000 7f8565740500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.593+0000 7f8565740500 1 NOTICE: finished shard 8 (0 entries found) 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.593+0000 7f8565740500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:30.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.593+0000 7f8565740500 1 NOTICE: finished shard 9 (0 entries found) 2023-09-22T13:04:30.597 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.593+0000 7f8565740500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:30.597 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.594+0000 7f8565740500 1 NOTICE: finished shard 10 (0 entries found) 2023-09-22T13:04:30.597 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.594+0000 7f8565740500 1 NOTICE: finished all shards (0 entries found) 2023-09-22T13:04:30.597 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.595+0000 7f8565740500 20 remove_watcher() i=0 2023-09-22T13:04:30.597 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.595+0000 7f8565740500 2 removed watcher, disabling cache 2023-09-22T13:04:30.597 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.596+0000 7f8565740500 20 remove_watcher() i=1 2023-09-22T13:04:30.598 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.597+0000 7f8565740500 20 remove_watcher() i=2 2023-09-22T13:04:30.599 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.598+0000 7f8565740500 20 remove_watcher() i=3 2023-09-22T13:04:30.600 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.599+0000 7f8565740500 20 remove_watcher() i=4 2023-09-22T13:04:30.601 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.600+0000 7f8565740500 20 remove_watcher() i=5 2023-09-22T13:04:30.602 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.601+0000 7f8565740500 20 remove_watcher() i=6 2023-09-22T13:04:30.603 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.603+0000 7f8565740500 20 remove_watcher() i=7 2023-09-22T13:04:30.604 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:30.603+0000 7f8521fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:30.608 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: verify that bucket check fixes do not affect null version objects 2023-09-22T13:04:30.608 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:30.660 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.660 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.660 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.660 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.660 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.660 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.660 INFO:tasks.workunit.client.0.smithi170.stderr:TEST: bucket check stats are correct in the presence of unlinked entries 2023-09-22T13:04:30.660 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:30.869 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.869 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.870 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.870 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.870 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.870 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.870 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.870 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.870 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.870 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.870 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.870 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.871 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.872 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.872 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.872 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:30.872 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:30.872 INFO:tasks.workunit.client.0.smithi170.stderr:ceph config set client rgw_debug_inject_set_olh_err 2 2023-09-22T13:04:31.165 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:31.165+0000 7fde89491700 0 --2- 172.21.15.170:0/232746134 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x7fde8413a700 0x7fde840a4e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:04:31.462 INFO:tasks.workunit.client.0.smithi170.stderr:ceph config set client rgw_debug_inject_olh_cancel_modification_err true 2023-09-22T13:04:32.994 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:32.994 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:32.994 INFO:tasks.workunit.client.0.smithi170.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:04:32.996 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:04:33.025 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:33.025 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:33.034 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.034+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.037 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.037+0000 7f41e3f31500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.037 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.037+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.038 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.037+0000 7f41e3f31500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.038 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.037+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.038 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.038+0000 7f41e3f31500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.038 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.038+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.239 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.239+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.254 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.253+0000 7f41e3f31500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:33.254 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.253+0000 7f41e3f31500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:33.254 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.253+0000 7f41e3f31500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:33.254 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.253+0000 7f41e3f31500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:33.254 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.253+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.254 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.253+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.254 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.253+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.255 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.254+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:33.255 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.254+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.255 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.254+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.255 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.254+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.255 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.254+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:33.256 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.254+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.256 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.255+0000 7f41e3f31500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.256 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.255+0000 7f41e3f31500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:33.256 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.255+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.255+0000 7f41e3f31500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.255+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.255+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.255+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.256+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:33.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.256+0000 7f41e3f31500 20 zonegroup default 2023-09-22T13:04:33.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.256+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.256+0000 7f41e3f31500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.256+0000 7f41e3f31500 10 Cannot find current period zone using local zone 2023-09-22T13:04:33.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.256+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.257 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.256+0000 7f41e3f31500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.258 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.256+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.258 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.257+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.257+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.257+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:33.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.257+0000 7f41e3f31500 20 zone default found 2023-09-22T13:04:33.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.257+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.257+0000 7f41e3f31500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.257+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.258+0000 7f41e3f31500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.258+0000 7f41e3f31500 20 started sync module instance, tier type = 2023-09-22T13:04:33.259 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.258+0000 7f41e3f31500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:33.268 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.267+0000 7f41cec12700 0 -- 172.21.15.170:0/888451753 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f41c402a400 msgr2=0x7f41c402a820 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:33.271 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.270+0000 7f41e3f31500 20 add_watcher() i=0 2023-09-22T13:04:33.271 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.270+0000 7f41e3f31500 20 add_watcher() i=1 2023-09-22T13:04:33.271 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.270+0000 7f41e3f31500 20 add_watcher() i=2 2023-09-22T13:04:33.271 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.270+0000 7f41e3f31500 20 add_watcher() i=3 2023-09-22T13:04:33.271 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.270+0000 7f41e3f31500 20 add_watcher() i=4 2023-09-22T13:04:33.271 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.270+0000 7f41e3f31500 20 add_watcher() i=5 2023-09-22T13:04:33.271 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.270+0000 7f41e3f31500 20 add_watcher() i=6 2023-09-22T13:04:33.271 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.271+0000 7f41e3f31500 20 add_watcher() i=7 2023-09-22T13:04:33.272 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.271+0000 7f41e3f31500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:33.274 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.273+0000 7f411bfff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:33.274 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.274+0000 7f41e3f31500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:33.275 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.274+0000 7f41e3f31500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:33.279 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.279+0000 7f41cf413700 0 -- 172.21.15.170:0/2757357750 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x55e579773f60 msgr2=0x55e57976f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:33.282 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.281+0000 7f41e3f31500 5 note: GC not initialized 2023-09-22T13:04:33.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.282+0000 7f410affd700 20 reqs_thread_entry: start 2023-09-22T13:04:33.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.282+0000 7f4108ff9700 20 reqs_thread_entry: start 2023-09-22T13:04:33.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.282+0000 7f41e3f31500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:33.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.283+0000 7f41e3f31500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:33.284 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.283+0000 7f4102ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:03 2023 2023-09-22T13:04:33.284 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:33.284 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.283+0000 7f41027fc700 20 reqs_thread_entry: start 2023-09-22T13:04:33.284 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.283+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e79b0 obj=default.rgw.meta:root:check-bucket state=0x55e5796f8e00 s->prefetch_data=0 2023-09-22T13:04:33.285 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.283+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.285 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.283+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.285 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.283+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:33.285 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.284+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e79b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.285 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.284+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.285 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.284+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.285 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.284+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.286 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.284+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.286 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.284+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.286 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.284+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.286 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.284+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e7780 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.285+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.285+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.285+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.285+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.287 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.285+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.285+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.285+0000 7f41e3f31500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:33.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.286+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e7780 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.286+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.286+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.286+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.286+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.286+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.286+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.286+0000 7f41e3f31500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:33.289 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.287+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e7780 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.289 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.288+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.289 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.288+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.290 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.288+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.290 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.288+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.290 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.288+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.290 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.288+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.290 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.288+0000 7f41e3f31500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:33.290 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.289+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e7780 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.289+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.289+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.289+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.289+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.290+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.290+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.290+0000 7f41e3f31500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:33.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.290+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e7780 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.292 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.291+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.292 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.291+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.292 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.291+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.292 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.291+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.291+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.291+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.291+0000 7f41e3f31500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:33.293 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.292+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e7780 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.292+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.292+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.292+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.292+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.293+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.293+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.293+0000 7f41e3f31500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:33.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.293+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e7780 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.293+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.293+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.293+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.293+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.294+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.294+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.294+0000 7f41e3f31500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:33.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.294+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e7780 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.295+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.295+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.295+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.296 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.295+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.295+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.295+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.295+0000 7f41e3f31500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:33.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.296+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e7780 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.297 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.297+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.298 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.297+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.298 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.297+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.298 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.297+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.298 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.297+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.299 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.297+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.299 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.297+0000 7f41e3f31500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:33.299 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.298+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e7780 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.299 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.298+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.299 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.298+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.299 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.298+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.299 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.298+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.298+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.298+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.298+0000 7f41e3f31500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:33.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.299+0000 7f41e3f31500 20 get_system_obj_state: rctx=0x7fff046e7780 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55e57966eb60 s->prefetch_data=0 2023-09-22T13:04:33.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.299+0000 7f41e3f31500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.299+0000 7f41e3f31500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.299+0000 7f41e3f31500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.299+0000 7f41e3f31500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.300+0000 7f41e3f31500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.300+0000 7f41e3f31500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.300+0000 7f41e3f31500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:33.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.302+0000 7f41e3f31500 20 remove_watcher() i=0 2023-09-22T13:04:33.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.302+0000 7f41e3f31500 2 removed watcher, disabling cache 2023-09-22T13:04:33.303 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.303+0000 7f41e3f31500 20 remove_watcher() i=1 2023-09-22T13:04:33.305 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.304+0000 7f41e3f31500 20 remove_watcher() i=2 2023-09-22T13:04:33.306 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.305+0000 7f41e3f31500 20 remove_watcher() i=3 2023-09-22T13:04:33.307 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.306+0000 7f41e3f31500 20 remove_watcher() i=4 2023-09-22T13:04:33.308 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.308+0000 7f41e3f31500 20 remove_watcher() i=5 2023-09-22T13:04:33.309 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.309+0000 7f41e3f31500 20 remove_watcher() i=6 2023-09-22T13:04:33.312 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.312+0000 7f41e3f31500 20 remove_watcher() i=7 2023-09-22T13:04:33.313 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.312+0000 7f411bfff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:33.377 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:33.377 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:33.377 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:33.377 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:33.377 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:33.377 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:33.377 INFO:tasks.workunit.client.0.smithi170.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:04:33.378 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:04:33.404 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:33.404 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:33.413 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.412+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.414 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.413+0000 7fc3bfe50500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.414 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.413+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.414+0000 7fc3bfe50500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.414+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.414+0000 7fc3bfe50500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.414+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.415 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.415+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.426 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.425+0000 7fc3bfe50500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:33.426 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.425+0000 7fc3bfe50500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:33.426 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.425+0000 7fc3bfe50500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:33.427 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.425+0000 7fc3bfe50500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:33.427 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.425+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.427 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.426+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.427 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.426+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.427 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.426+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:33.427 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.426+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.428 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.426+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.428 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.426+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.428 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.426+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:33.428 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.426+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.428 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.427+0000 7fc3bfe50500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.427+0000 7fc3bfe50500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.427+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.427+0000 7fc3bfe50500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.427+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.427+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.427+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.427+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.427+0000 7fc3bfe50500 20 zonegroup default 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.427+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.428+0000 7fc3bfe50500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.428+0000 7fc3bfe50500 10 Cannot find current period zone using local zone 2023-09-22T13:04:33.429 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.428+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.430 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.428+0000 7fc3bfe50500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.430 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.428+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.428+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.428+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.428+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:33.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.428+0000 7fc3bfe50500 20 zone default found 2023-09-22T13:04:33.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.428+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.429+0000 7fc3bfe50500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.429+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.429+0000 7fc3bfe50500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.429+0000 7fc3bfe50500 20 started sync module instance, tier type = 2023-09-22T13:04:33.431 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.429+0000 7fc3bfe50500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:33.446 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.445+0000 7fc3aab31700 0 -- 172.21.15.170:0/4248173630 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7fc39c029950 msgr2=0x7fc39c0283d0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:33.446 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.446+0000 7fc3bfe50500 20 add_watcher() i=0 2023-09-22T13:04:33.447 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.446+0000 7fc3bfe50500 20 add_watcher() i=1 2023-09-22T13:04:33.447 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.446+0000 7fc3bfe50500 20 add_watcher() i=2 2023-09-22T13:04:33.447 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.446+0000 7fc3bfe50500 20 add_watcher() i=3 2023-09-22T13:04:33.447 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.446+0000 7fc3bfe50500 20 add_watcher() i=4 2023-09-22T13:04:33.447 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.446+0000 7fc3bfe50500 20 add_watcher() i=5 2023-09-22T13:04:33.447 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.446+0000 7fc3bfe50500 20 add_watcher() i=6 2023-09-22T13:04:33.448 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.448+0000 7fc3bfe50500 20 add_watcher() i=7 2023-09-22T13:04:33.448 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.448+0000 7fc3bfe50500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:33.450 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.450+0000 7fc2f7fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:33.451 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.450+0000 7fc3bfe50500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:33.451 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.450+0000 7fc3bfe50500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:33.455 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.455+0000 7fc3ab332700 0 -- 172.21.15.170:0/4021239673 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x5599bdff0930 msgr2=0x5599bdfe7df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:33.457 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.457+0000 7fc3bfe50500 5 note: GC not initialized 2023-09-22T13:04:33.459 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.458+0000 7fc2eeffd700 20 reqs_thread_entry: start 2023-09-22T13:04:33.459 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.458+0000 7fc2ecff9700 20 reqs_thread_entry: start 2023-09-22T13:04:33.459 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.458+0000 7fc3bfe50500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:33.459 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.458+0000 7fc3bfe50500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:33.459 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.458+0000 7fc2daffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:03 2023 2023-09-22T13:04:33.459 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:33.459 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.458+0000 7fc2da7fc700 20 reqs_thread_entry: start 2023-09-22T13:04:33.459 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.458+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c330 obj=default.rgw.meta:root:check-bucket state=0x5599bdfcbf30 s->prefetch_data=0 2023-09-22T13:04:33.460 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.458+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.460 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.458+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.461 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.459+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:33.461 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.459+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c330 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.461 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.459+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.461 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.459+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.461 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.459+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.461 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.459+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.462 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.460+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.462 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.460+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.462 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.460+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c100 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.462 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.460+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.462 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.460+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.462 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.460+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.462 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.460+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.462 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.460+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.462 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.460+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.462 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.460+0000 7fc3bfe50500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:33.462 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.461+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c100 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.464 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.461+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.464 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.461+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.464 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.461+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.464 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.461+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.465 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.461+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.465 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.461+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.465 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.461+0000 7fc3bfe50500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:33.465 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.462+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c100 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.465 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.462+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.465 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.462+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.465 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.462+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.465 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.462+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.465 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.463+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.465 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.463+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.465 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.463+0000 7fc3bfe50500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:33.466 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.463+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c100 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.466 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.463+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.466 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.463+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.466 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.463+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.466 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.463+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.466 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.464+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.466 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.464+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.466 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.464+0000 7fc3bfe50500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:33.467 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.465+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c100 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.467 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.465+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.467 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.465+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.465+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.465+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.465+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.465+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.465+0000 7fc3bfe50500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.466+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c100 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.466+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.466+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.466+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.466+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.466+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.466+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.469 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.466+0000 7fc3bfe50500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:33.469 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.467+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c100 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.470 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.467+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.470 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.467+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.470 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.467+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.470 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.467+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.470 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.467+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.470 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.467+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.470 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.467+0000 7fc3bfe50500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:33.470 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.468+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c100 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.470 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.468+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.471 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.468+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.471 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.468+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.471 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.468+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.471 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.468+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.471 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.469+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.471 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.469+0000 7fc3bfe50500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:33.471 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.469+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c100 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.473 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.470+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.473 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.470+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.473 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.470+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.473 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.470+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.473 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.470+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.473 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.470+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.473 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.470+0000 7fc3bfe50500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:33.473 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.470+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c100 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.473 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.471+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.474 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.471+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.474 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.471+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.474 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.471+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.474 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.471+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.474 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.471+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.474 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.471+0000 7fc3bfe50500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:33.474 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.472+0000 7fc3bfe50500 20 get_system_obj_state: rctx=0x7ffe3bb0c100 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5599bdeeab60 s->prefetch_data=0 2023-09-22T13:04:33.474 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.472+0000 7fc3bfe50500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.474 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.472+0000 7fc3bfe50500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.474 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.472+0000 7fc3bfe50500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.474 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.472+0000 7fc3bfe50500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.475 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.472+0000 7fc3bfe50500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.475 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.472+0000 7fc3bfe50500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.475 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.472+0000 7fc3bfe50500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:33.475 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.474+0000 7fc3bfe50500 20 remove_watcher() i=0 2023-09-22T13:04:33.475 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.474+0000 7fc3bfe50500 2 removed watcher, disabling cache 2023-09-22T13:04:33.475 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.475+0000 7fc3bfe50500 20 remove_watcher() i=1 2023-09-22T13:04:33.477 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.476+0000 7fc3bfe50500 20 remove_watcher() i=2 2023-09-22T13:04:33.478 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.477+0000 7fc3bfe50500 20 remove_watcher() i=3 2023-09-22T13:04:33.479 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.478+0000 7fc3bfe50500 20 remove_watcher() i=4 2023-09-22T13:04:33.480 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.479+0000 7fc3bfe50500 20 remove_watcher() i=5 2023-09-22T13:04:33.481 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.480+0000 7fc3bfe50500 20 remove_watcher() i=6 2023-09-22T13:04:33.483 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.483+0000 7fc3bfe50500 20 remove_watcher() i=7 2023-09-22T13:04:33.484 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.484+0000 7fc2f7fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:33.544 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:33.544 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:33.544 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:33.544 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:33.544 INFO:tasks.workunit.client.0.smithi170.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:04:33.545 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object e 2023-09-22T13:04:33.569 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:33.569 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:33.577 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.576+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.578 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.578+0000 7fc6f9aef500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.579 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.578+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.579 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.579+0000 7fc6f9aef500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.579 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.579+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.580 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.579+0000 7fc6f9aef500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.580 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.579+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.580 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.580+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.590+0000 7fc6f9aef500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:33.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.590+0000 7fc6f9aef500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:33.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.590+0000 7fc6f9aef500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:33.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.590+0000 7fc6f9aef500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:33.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.590+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.590+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.590+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.591 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.590+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:33.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.590+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.590+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.592 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.590+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.594 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.591+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:33.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.591+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.591+0000 7fc6f9aef500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.591+0000 7fc6f9aef500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:33.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.591+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.591+0000 7fc6f9aef500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.591+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.591+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.591+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.592+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:33.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.592+0000 7fc6f9aef500 20 zonegroup default 2023-09-22T13:04:33.595 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.592+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.592+0000 7fc6f9aef500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.592+0000 7fc6f9aef500 10 Cannot find current period zone using local zone 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.592+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.592+0000 7fc6f9aef500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.592+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.592+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.592+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.593+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.593+0000 7fc6f9aef500 20 zone default found 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.593+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.593+0000 7fc6f9aef500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.596 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.593+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.597 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.593+0000 7fc6f9aef500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.597 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.593+0000 7fc6f9aef500 20 started sync module instance, tier type = 2023-09-22T13:04:33.597 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.593+0000 7fc6f9aef500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:33.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.615+0000 7fc6f9aef500 20 add_watcher() i=0 2023-09-22T13:04:33.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.615+0000 7fc6f9aef500 20 add_watcher() i=1 2023-09-22T13:04:33.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.615+0000 7fc6f9aef500 20 add_watcher() i=2 2023-09-22T13:04:33.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.615+0000 7fc6f9aef500 20 add_watcher() i=3 2023-09-22T13:04:33.615 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.615+0000 7fc6f9aef500 20 add_watcher() i=4 2023-09-22T13:04:33.616 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.615+0000 7fc6f9aef500 20 add_watcher() i=5 2023-09-22T13:04:33.616 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.615+0000 7fc6f9aef500 20 add_watcher() i=6 2023-09-22T13:04:33.616 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.616+0000 7fc6f9aef500 20 add_watcher() i=7 2023-09-22T13:04:33.616 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.616+0000 7fc6f9aef500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:33.616 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.616+0000 7fc6e57d2700 0 -- 172.21.15.170:0/2295514119 >> [v2:172.21.15.191:6816/144992,v1:172.21.15.191:6817/144992] conn(0x7fc6d804db40 msgr2=0x7fc6d804b000 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:33.620 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.620+0000 7fc6b57ea700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:33.621 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.620+0000 7fc6f9aef500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:33.621 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.620+0000 7fc6f9aef500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:33.624 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.624+0000 7fc6e57d2700 0 -- 172.21.15.170:0/564712585 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x55780d68adc0 msgr2=0x55780d6bb560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:33.825 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.825+0000 7fc6e57d2700 0 --2- 172.21.15.170:0/564712585 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x55780d68adc0 0x55780d6bb560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:04:33.828 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.827+0000 7fc6f9aef500 5 note: GC not initialized 2023-09-22T13:04:33.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.828+0000 7fc623fff700 20 reqs_thread_entry: start 2023-09-22T13:04:33.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.828+0000 7fc621ffb700 20 reqs_thread_entry: start 2023-09-22T13:04:33.829 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.828+0000 7fc6f9aef500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:33.830 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.828+0000 7fc6f9aef500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:33.830 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.828+0000 7fc60ffff700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:04 2023 2023-09-22T13:04:33.830 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:33.830 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.828+0000 7fc60f7fe700 20 reqs_thread_entry: start 2023-09-22T13:04:33.830 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.828+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f54c0 obj=default.rgw.meta:root:check-bucket state=0x55780d653d00 s->prefetch_data=0 2023-09-22T13:04:33.832 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.829+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.833 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.829+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.833 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.829+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:33.833 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.829+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f54c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.833 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.830+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.833 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.830+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.833 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.830+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.833 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.830+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.833 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.830+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.833 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.830+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.834 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.830+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f5290 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.834 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.830+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.834 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.830+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.834 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.830+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.834 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.830+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.834 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.831+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.834 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.831+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.834 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.831+0000 7fc6f9aef500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:33.834 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.831+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f5290 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.835 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.831+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.835 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.831+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.835 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.831+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.835 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.831+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.835 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.832+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.835 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.832+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.835 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.832+0000 7fc6f9aef500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:33.838 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.833+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f5290 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.838 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.833+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.838 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.833+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.838 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.833+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.838 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.833+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.839 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.833+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.839 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.833+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.839 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.833+0000 7fc6f9aef500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:33.839 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.833+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f5290 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.839 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.834+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.839 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.834+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.839 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.834+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.839 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.834+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.839 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.834+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.839 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.834+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.840 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.834+0000 7fc6f9aef500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:33.840 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.835+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f5290 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.840 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.835+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.840 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.835+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.840 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.835+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.840 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.835+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.840 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.835+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.840 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.835+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.840 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.835+0000 7fc6f9aef500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:33.841 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.836+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f5290 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.841 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.836+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.841 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.836+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.841 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.836+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.841 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.836+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.841 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.836+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.841 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.836+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.841 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.836+0000 7fc6f9aef500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:33.841 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.837+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f5290 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.837+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.837+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.837+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.837+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.837+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.837+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.842 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.837+0000 7fc6f9aef500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:33.844 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.838+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f5290 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.844 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.838+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.844 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.838+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.844 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.838+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.845 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.838+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.845 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.838+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.845 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.838+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.845 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.838+0000 7fc6f9aef500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:33.845 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.839+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f5290 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.845 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.839+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.845 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.839+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.845 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.839+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.845 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.839+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.845 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.840+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.840+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.840+0000 7fc6f9aef500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:33.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.840+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f5290 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.840+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.840+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.840+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.840+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.841+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.841+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.846 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.841+0000 7fc6f9aef500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:33.847 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.841+0000 7fc6f9aef500 20 get_system_obj_state: rctx=0x7ffe320f5290 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x55780d5adb60 s->prefetch_data=0 2023-09-22T13:04:33.847 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.842+0000 7fc6f9aef500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:33.847 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.842+0000 7fc6f9aef500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:33.847 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.842+0000 7fc6f9aef500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:33.847 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.842+0000 7fc6f9aef500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.847 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.842+0000 7fc6f9aef500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:33.847 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.842+0000 7fc6f9aef500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:33.847 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.842+0000 7fc6f9aef500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:33.847 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.844+0000 7fc6f9aef500 20 remove_watcher() i=0 2023-09-22T13:04:33.847 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.844+0000 7fc6f9aef500 2 removed watcher, disabling cache 2023-09-22T13:04:33.848 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.845+0000 7fc6f9aef500 20 remove_watcher() i=1 2023-09-22T13:04:33.848 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.846+0000 7fc6f9aef500 20 remove_watcher() i=2 2023-09-22T13:04:33.848 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.847+0000 7fc6f9aef500 20 remove_watcher() i=3 2023-09-22T13:04:33.849 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.848+0000 7fc6f9aef500 20 remove_watcher() i=4 2023-09-22T13:04:33.850 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.849+0000 7fc6f9aef500 20 remove_watcher() i=5 2023-09-22T13:04:33.851 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.850+0000 7fc6f9aef500 20 remove_watcher() i=6 2023-09-22T13:04:33.852 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.851+0000 7fc6f9aef500 20 remove_watcher() i=7 2023-09-22T13:04:33.852 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.852+0000 7fc6b57ea700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:33.913 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:33.913 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:33.913 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:33.913 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:33.913 INFO:tasks.workunit.client.0.smithi170.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:04:33.914 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bi list --bucket check-bucket --object f 2023-09-22T13:04:33.940 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:33.940 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:33.947 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.947+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.949 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.948+0000 7f760e584500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.949 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.948+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.950 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.949+0000 7f760e584500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.950 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.949+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.950 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.949+0000 7f760e584500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.950 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.949+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.951 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.950+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.961 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.960+0000 7f760e584500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:33.961 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.960+0000 7f760e584500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:33.961 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.960+0000 7f760e584500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:33.961 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.960+0000 7f760e584500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:33.961 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.960+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.961 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.960+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.961 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.960+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.961 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.960+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:33.962 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.960+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.962 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.960+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.962 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.960+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.964 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.961+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:33.964 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.961+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.964 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.961+0000 7f760e584500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.964 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.961+0000 7f760e584500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:33.964 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.961+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.965 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.961+0000 7f760e584500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.965 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.961+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.965 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.961+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.965 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.961+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.965 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.962+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:33.965 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.962+0000 7f760e584500 20 zonegroup default 2023-09-22T13:04:33.965 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.962+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.965 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.962+0000 7f760e584500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.965 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.962+0000 7f760e584500 10 Cannot find current period zone using local zone 2023-09-22T13:04:33.965 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.962+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.962+0000 7f760e584500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.962+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.963+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:33.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.963+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.963+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:33.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.963+0000 7f760e584500 20 zone default found 2023-09-22T13:04:33.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.963+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.963+0000 7f760e584500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.963+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:33.966 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.963+0000 7f760e584500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:33.967 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.963+0000 7f760e584500 20 started sync module instance, tier type = 2023-09-22T13:04:33.967 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.963+0000 7f760e584500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:33.984 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.984+0000 7f75f9265700 0 -- 172.21.15.170:0/455288904 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f75ec021460 msgr2=0x7f75ec0218a0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:33.984 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.984+0000 7f760e584500 20 add_watcher() i=0 2023-09-22T13:04:33.984 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.984+0000 7f760e584500 20 add_watcher() i=1 2023-09-22T13:04:33.984 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.984+0000 7f760e584500 20 add_watcher() i=2 2023-09-22T13:04:33.984 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.984+0000 7f760e584500 20 add_watcher() i=3 2023-09-22T13:04:33.984 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.984+0000 7f760e584500 20 add_watcher() i=4 2023-09-22T13:04:33.985 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.984+0000 7f760e584500 20 add_watcher() i=5 2023-09-22T13:04:33.985 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.984+0000 7f760e584500 20 add_watcher() i=6 2023-09-22T13:04:33.989 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.988+0000 7f760e584500 20 add_watcher() i=7 2023-09-22T13:04:33.989 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.988+0000 7f760e584500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:33.991 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.990+0000 7f75cb7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:33.991 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.991+0000 7f760e584500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:33.991 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.991+0000 7f760e584500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:33.995 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.995+0000 7f75f9a66700 0 -- 172.21.15.170:0/1396493429 >> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] conn(0x562919443e40 msgr2=0x562919436030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:33.997 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.997+0000 7f760e584500 5 note: GC not initialized 2023-09-22T13:04:33.999 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.998+0000 7f753a7fc700 20 reqs_thread_entry: start 2023-09-22T13:04:33.999 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.998+0000 7f752bfff700 20 reqs_thread_entry: start 2023-09-22T13:04:33.999 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.998+0000 7f760e584500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:33.999 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.998+0000 7f760e584500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:33.999 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.998+0000 7f752a7fc700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:04 2023 2023-09-22T13:04:34.000 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:34.000 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.998+0000 7f7529ffb700 20 reqs_thread_entry: start 2023-09-22T13:04:34.000 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.998+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b765f0 obj=default.rgw.meta:root:check-bucket state=0x562919445b80 s->prefetch_data=0 2023-09-22T13:04:34.000 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.999+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.000 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.999+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.002 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.999+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:34.002 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.999+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b765f0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.002 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.999+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.003 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.999+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.003 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.999+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.003 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.999+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.003 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:33.999+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.003 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.000+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.003 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.000+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b763c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.003 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.000+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.003 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.000+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.003 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.000+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.003 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.000+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.004 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.000+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.004 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.000+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.004 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.000+0000 7f760e584500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:34.004 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.000+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b763c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.004 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.001+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.004 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.001+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.004 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.001+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.004 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.001+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.004 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.001+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.005 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.001+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.005 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.001+0000 7f760e584500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:34.005 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.002+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b763c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.008 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.002+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.008 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.002+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.009 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.002+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.009 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.002+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.009 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.002+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.009 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.002+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.009 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.002+0000 7f760e584500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:34.009 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.003+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b763c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.009 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.003+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.009 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.003+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.010 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.003+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.010 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.003+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.010 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.003+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.010 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.003+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.010 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.003+0000 7f760e584500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:34.010 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.004+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b763c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.010 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.004+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.010 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.004+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.010 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.004+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.010 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.004+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.011 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.004+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.011 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.004+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.011 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.004+0000 7f760e584500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:34.011 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.005+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b763c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.011 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.005+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.011 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.005+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.011 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.005+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.011 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.005+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.011 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.005+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.011 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.005+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.012 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.005+0000 7f760e584500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:34.012 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.006+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b763c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.012 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.006+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.012 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.006+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.012 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.006+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.012 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.006+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.012 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.006+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.012 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.006+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.012 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.006+0000 7f760e584500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:34.012 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.007+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b763c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.013 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.007+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.013 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.007+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.013 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.007+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.013 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.007+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.013 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.007+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.013 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.007+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.013 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.007+0000 7f760e584500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:34.013 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.008+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b763c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.015 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.008+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.016 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.008+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.016 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.008+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.016 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.008+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.016 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.009+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.016 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.009+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.016 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.009+0000 7f760e584500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:34.016 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.009+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b763c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.016 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.009+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.016 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.009+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.017 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.009+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.017 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.009+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.017 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.010+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.017 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.010+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.017 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.010+0000 7f760e584500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:34.017 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.010+0000 7f760e584500 20 get_system_obj_state: rctx=0x7fffc9b763c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x562919337b60 s->prefetch_data=0 2023-09-22T13:04:34.017 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.011+0000 7f760e584500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:34.018 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.011+0000 7f760e584500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:34.018 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.011+0000 7f760e584500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:34.018 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.011+0000 7f760e584500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.018 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.011+0000 7f760e584500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:34.018 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.011+0000 7f760e584500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:34.018 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.011+0000 7f760e584500 20 bucket index oid: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:34.018 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.013+0000 7f760e584500 20 remove_watcher() i=0 2023-09-22T13:04:34.018 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.013+0000 7f760e584500 2 removed watcher, disabling cache 2023-09-22T13:04:34.018 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.014+0000 7f760e584500 20 remove_watcher() i=1 2023-09-22T13:04:34.019 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.017+0000 7f760e584500 20 remove_watcher() i=2 2023-09-22T13:04:34.019 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.018+0000 7f760e584500 20 remove_watcher() i=3 2023-09-22T13:04:34.019 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.019+0000 7f760e584500 20 remove_watcher() i=4 2023-09-22T13:04:34.021 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.020+0000 7f760e584500 20 remove_watcher() i=5 2023-09-22T13:04:34.022 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.021+0000 7f760e584500 20 remove_watcher() i=6 2023-09-22T13:04:34.023 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.022+0000 7f760e584500 20 remove_watcher() i=7 2023-09-22T13:04:34.023 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.023+0000 7f75cb7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:34.034 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:34.034 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:34.034 INFO:tasks.workunit.client.0.smithi170.stderr:ceph config rm client rgw_debug_inject_set_olh_err 2023-09-22T13:04:34.441 INFO:tasks.workunit.client.0.smithi170.stderr:ceph config rm client rgw_debug_inject_olh_cancel_modification_err 2023-09-22T13:04:34.855 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bucket check --fix --bucket check-bucket 2023-09-22T13:04:34.878 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:34.878 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:34.886 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.885+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.888 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.887+0000 7f3d1d765500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:34.888 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.887+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.889 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.888+0000 7f3d1d765500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:34.889 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.888+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.889 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.888+0000 7f3d1d765500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:34.889 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.888+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.889 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.889+0000 7f3d1d765500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:34.900 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.899+0000 7f3d1d765500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:34.900 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.899+0000 7f3d1d765500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:34.900 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.899+0000 7f3d1d765500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:34.900 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.899+0000 7f3d1d765500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:34.900 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.899+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.900 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.899+0000 7f3d1d765500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:34.900 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.899+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.902 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.900+0000 7f3d1d765500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:34.902 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.900+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.902 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.900+0000 7f3d1d765500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:34.902 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.900+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.902 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.900+0000 7f3d1d765500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:34.902 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.900+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.902 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.900+0000 7f3d1d765500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:34.903 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.900+0000 7f3d1d765500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:34.903 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.900+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.903 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.901+0000 7f3d1d765500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:34.903 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.901+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.903 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.901+0000 7f3d1d765500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:34.903 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.901+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.903 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.901+0000 7f3d1d765500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:34.903 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.901+0000 7f3d1d765500 20 zonegroup default 2023-09-22T13:04:34.903 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.901+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.903 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:34.904 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 10 Cannot find current period zone using local zone 2023-09-22T13:04:34.904 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.904 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:34.905 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.905 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:34.905 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.905 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:34.905 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 20 zone default found 2023-09-22T13:04:34.905 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.905 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:34.905 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.902+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:34.905 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.903+0000 7f3d1d765500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:34.905 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.903+0000 7f3d1d765500 20 started sync module instance, tier type = 2023-09-22T13:04:34.906 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.903+0000 7f3d1d765500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:34.913 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.912+0000 7f3d03fff700 0 -- 172.21.15.170:0/1236411023 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f3cf404db60 msgr2=0x7f3cf404b020 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:34.913 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.912+0000 7f3d1d765500 20 add_watcher() i=0 2023-09-22T13:04:34.913 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.912+0000 7f3d1d765500 20 add_watcher() i=1 2023-09-22T13:04:34.913 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.912+0000 7f3d1d765500 20 add_watcher() i=2 2023-09-22T13:04:34.913 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.912+0000 7f3d1d765500 20 add_watcher() i=3 2023-09-22T13:04:34.913 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.912+0000 7f3d1d765500 20 add_watcher() i=4 2023-09-22T13:04:34.913 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.912+0000 7f3d1d765500 20 add_watcher() i=5 2023-09-22T13:04:34.913 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:34.912+0000 7f3d1d765500 20 add_watcher() i=6 2023-09-22T13:04:35.116 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.116+0000 7f3d1d765500 20 add_watcher() i=7 2023-09-22T13:04:35.116 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.116+0000 7f3d1d765500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:35.118 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.118+0000 7f3cd9fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:35.119 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.118+0000 7f3d1d765500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:35.119 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.118+0000 7f3d1d765500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:35.122 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.121+0000 7f3d03fff700 0 -- >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x561c5b5c45e0 msgr2=0x561c5b5d2970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:35.124 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.124+0000 7f3d1d765500 5 note: GC not initialized 2023-09-22T13:04:35.125 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.125+0000 7f3c50ff9700 20 reqs_thread_entry: start 2023-09-22T13:04:35.126 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.125+0000 7f3c467fc700 20 reqs_thread_entry: start 2023-09-22T13:04:35.126 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.125+0000 7f3d1d765500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:35.126 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.125+0000 7f3d1d765500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:35.126 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.125+0000 7f3c44ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:05 2023 2023-09-22T13:04:35.126 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:35.126 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.125+0000 7f3c2bfff700 20 reqs_thread_entry: start 2023-09-22T13:04:35.126 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.125+0000 7f3d1d765500 20 get_system_obj_state: rctx=0x7ffc22ad56d8 obj=default.rgw.meta:root:check-bucket state=0x561c5b55faa0 s->prefetch_data=0 2023-09-22T13:04:35.126 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.125+0000 7f3d1d765500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:04:35.128 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:04:35.128 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:35.128 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:35.128 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:35.128 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:04:35.128 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.128 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:35.128 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:04:35.128 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:35.128 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 20 get_system_obj_state: rctx=0x7ffc22ad57a8 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x561c5b55faa0 s->prefetch_data=0 2023-09-22T13:04:35.128 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : miss 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x16 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.126+0000 7f3d1d765500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.127+0000 7f3d1d765500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.127+0000 7f3d1d765500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x13 2023-09-22T13:04:35.129 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.127+0000 7f3d1d765500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:35.130 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.127+0000 7f3d1d765500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:35.130 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.127+0000 7f3d1d765500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:35.130 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.127+0000 7f3d1d765500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 2023-09-22T13:04:35.130 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.127+0000 7f3d1d765500 20 RGWRados::Bucket::List::list_objects_ordered starting attempt 1 2023-09-22T13:04:35.130 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.127+0000 7f3d1d765500 10 RGWRados::cls_bucket_list_ordered: :check-bucket[772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1]) start_after="_multipart_[]", prefix="_multipart_" num_entries=1001, list_versions=1, expansion_factor=1 2023-09-22T13:04:35.130 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.127+0000 7f3d1d765500 10 RGWRados::cls_bucket_list_ordered request from each of 11 shard(s) for 112 entries to get 1001 total entries 2023-09-22T13:04:35.130 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.128+0000 7f3d1d765500 20 RGWRados::cls_bucket_list_ordered: returning, count=0, is_truncated=0 2023-09-22T13:04:35.131 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.128+0000 7f3d1d765500 20 RGWRados::cls_bucket_list_ordered: returning, last_entry NOT SET 2023-09-22T13:04:35.131 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.128+0000 7f3d1d765500 20 RGWRados::Bucket::List::list_objects_ordered INFO end of outer loop, truncated=0, count=0, attempt=1 2023-09-22T13:04:35.134 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.133+0000 7f3d1d765500 20 remove_watcher() i=0 2023-09-22T13:04:35.134 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.133+0000 7f3d1d765500 2 removed watcher, disabling cache 2023-09-22T13:04:35.135 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.134+0000 7f3d1d765500 20 remove_watcher() i=1 2023-09-22T13:04:35.136 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.136+0000 7f3d1d765500 20 remove_watcher() i=2 2023-09-22T13:04:35.137 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.137+0000 7f3d1d765500 20 remove_watcher() i=3 2023-09-22T13:04:35.138 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.138+0000 7f3d1d765500 20 remove_watcher() i=4 2023-09-22T13:04:35.139 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.139+0000 7f3d1d765500 20 remove_watcher() i=5 2023-09-22T13:04:35.140 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.140+0000 7f3d1d765500 20 remove_watcher() i=6 2023-09-22T13:04:35.141 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.141+0000 7f3d1d765500 20 remove_watcher() i=7 2023-09-22T13:04:35.142 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.141+0000 7f3cd9fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:35.148 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys 2023-09-22T13:04:35.177 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:35.177 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:35.185 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.185+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.187 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.187+0000 7f42f1ebc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.187 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.187+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.187+0000 7f42f1ebc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.187+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.188+0000 7f42f1ebc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.188 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.188+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.189 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.188+0000 7f42f1ebc500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:35.199 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.198+0000 7f42f1ebc500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:35.199 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.198+0000 7f42f1ebc500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:35.199 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.198+0000 7f42f1ebc500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:35.199 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.198+0000 7f42f1ebc500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:35.199 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.198+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.199 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.198+0000 7f42f1ebc500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:35.199 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.198+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.201 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.199+0000 7f42f1ebc500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:35.201 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.199+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.201 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.199+0000 7f42f1ebc500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:35.201 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.199+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.201 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.199+0000 7f42f1ebc500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:35.201 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.199+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.201 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.199+0000 7f42f1ebc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.201 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.199+0000 7f42f1ebc500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:35.202 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.199+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.202 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.200+0000 7f42f1ebc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.202 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.200+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.202 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.200+0000 7f42f1ebc500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:35.202 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.200+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.202 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.200+0000 7f42f1ebc500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:35.202 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.200+0000 7f42f1ebc500 20 zonegroup default 2023-09-22T13:04:35.202 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.200+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.202 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.200+0000 7f42f1ebc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.202 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.200+0000 7f42f1ebc500 10 Cannot find current period zone using local zone 2023-09-22T13:04:35.202 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.200+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.204 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.201+0000 7f42f1ebc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.204 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.201+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.204 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.201+0000 7f42f1ebc500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:35.204 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.201+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.204 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.201+0000 7f42f1ebc500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:35.204 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.201+0000 7f42f1ebc500 20 zone default found 2023-09-22T13:04:35.204 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.201+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.204 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.202+0000 7f42f1ebc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.204 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.202+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.204 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.202+0000 7f42f1ebc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.204 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.202+0000 7f42f1ebc500 20 started sync module instance, tier type = 2023-09-22T13:04:35.205 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.202+0000 7f42f1ebc500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:35.218 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.217+0000 7f42f1ebc500 20 add_watcher() i=0 2023-09-22T13:04:35.218 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.217+0000 7f42f1ebc500 20 add_watcher() i=1 2023-09-22T13:04:35.218 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.217+0000 7f42f1ebc500 20 add_watcher() i=2 2023-09-22T13:04:35.218 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.217+0000 7f42f1ebc500 20 add_watcher() i=3 2023-09-22T13:04:35.218 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.217+0000 7f42f1ebc500 20 add_watcher() i=4 2023-09-22T13:04:35.218 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.217+0000 7f42f1ebc500 20 add_watcher() i=5 2023-09-22T13:04:35.218 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.217+0000 7f42f1ebc500 20 add_watcher() i=6 2023-09-22T13:04:35.220 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.220+0000 7f42f1ebc500 20 add_watcher() i=7 2023-09-22T13:04:35.221 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.220+0000 7f42f1ebc500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:35.221 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.220+0000 7f42ddb9f700 0 -- 172.21.15.170:0/407311147 >> [v2:172.21.15.191:6816/144992,v1:172.21.15.191:6817/144992] conn(0x7f42d0031410 msgr2=0x7f42d0031850 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:35.223 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.223+0000 7f42ae7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:35.224 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.223+0000 7f42f1ebc500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:35.224 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.223+0000 7f42f1ebc500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:35.228 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.228+0000 7f42dd39e700 0 -- 172.21.15.170:0/885357989 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x5641384238e0 msgr2=0x564138419f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:35.230 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.230+0000 7f42f1ebc500 5 note: GC not initialized 2023-09-22T13:04:35.232 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.231+0000 7f421d7fa700 20 reqs_thread_entry: start 2023-09-22T13:04:35.232 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.231+0000 7f4216ffd700 20 reqs_thread_entry: start 2023-09-22T13:04:35.232 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.231+0000 7f42f1ebc500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:35.232 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.231+0000 7f42f1ebc500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:35.232 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.231+0000 7f42157fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:05 2023 2023-09-22T13:04:35.232 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:35.232 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.231+0000 7f4214ff9700 20 reqs_thread_entry: start 2023-09-22T13:04:35.233 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.231+0000 7f42f1ebc500 20 get_system_obj_state: rctx=0x7fff3caac878 obj=default.rgw.meta:root:check-bucket state=0x5641383b7b80 s->prefetch_data=0 2023-09-22T13:04:35.233 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.231+0000 7f42f1ebc500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:04:35.234 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.232+0000 7f42f1ebc500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:04:35.234 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.232+0000 7f42f1ebc500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:35.234 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.232+0000 7f42f1ebc500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:35.234 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.232+0000 7f42f1ebc500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:35.235 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.232+0000 7f42f1ebc500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:04:35.235 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.232+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.235 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.232+0000 7f42f1ebc500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:35.235 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.232+0000 7f42f1ebc500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:04:35.235 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.232+0000 7f42f1ebc500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:04:35.235 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.232+0000 7f42f1ebc500 20 get_system_obj_state: rctx=0x7fff3caac948 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5641383b7b80 s->prefetch_data=0 2023-09-22T13:04:35.235 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.232+0000 7f42f1ebc500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : miss 2023-09-22T13:04:35.235 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x16 2023-09-22T13:04:35.235 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:35.235 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:35.235 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 info.flags=0x13 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 to cache LRU end 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 2023-09-22T13:04:35.236 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.233+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.0 2023-09-22T13:04:35.237 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.234+0000 7f42f1ebc500 1 NOTICE: finished shard 0 (0 entries removed) 2023-09-22T13:04:35.237 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.234+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:35.238 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.234+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.2 2023-09-22T13:04:35.238 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.235+0000 7f42f1ebc500 1 NOTICE: finished shard 2 (0 entries removed) 2023-09-22T13:04:35.238 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.235+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.3 2023-09-22T13:04:35.238 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.235+0000 7f42f1ebc500 1 NOTICE: finished shard 3 (0 entries removed) 2023-09-22T13:04:35.238 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.235+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.4 2023-09-22T13:04:35.239 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.235+0000 7f42f1ebc500 1 NOTICE: finished shard 4 (0 entries removed) 2023-09-22T13:04:35.239 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.235+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:35.239 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.236+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.6 2023-09-22T13:04:35.239 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.236+0000 7f42f1ebc500 1 NOTICE: finished shard 6 (0 entries removed) 2023-09-22T13:04:35.239 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.236+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.7 2023-09-22T13:04:35.239 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.237+0000 7f42f1ebc500 1 NOTICE: finished shard 7 (0 entries removed) 2023-09-22T13:04:35.239 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.237+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:35.239 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.237+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.9 2023-09-22T13:04:35.239 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.237+0000 7f42f1ebc500 1 NOTICE: finished shard 9 (0 entries removed) 2023-09-22T13:04:35.239 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.237+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:35.240 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.238+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:35.240 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.238+0000 7f42f1ebc500 20 get_obj_state: rctx=0x5641384b4370 obj=check-bucket:e state=0x5641384db2b8 s->prefetch_data=0 2023-09-22T13:04:35.240 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.239+0000 7f42f1ebc500 20 get_obj_state: setting s->obj_tag to jstt76wc1vwmhnd0vnd4kypzm721imk7 2023-09-22T13:04:35.240 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.239+0000 7f42f1ebc500 20 get_obj_state_impl: setting s->olh_tag to ejvwkqja5a1thiwjjjy7wxf382n40id5 2023-09-22T13:04:35.240 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.239+0000 7f42f1ebc500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=ejvwkqja5a1thiwjjjy7wxf382n40id5 2023-09-22T13:04:35.241 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.240+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:35.242 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.242+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:35.243 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.242+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:35.243 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.242+0000 7f42f1ebc500 20 olh_log_entry: epoch=2 op=2 key=e[] 2023-09-22T13:04:35.244 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.242+0000 7f42f1ebc500 20 olh_log_entry: epoch=2 op=3 key=e[ANUZGLNiZ5Cy6OE81kEqC-8tsfsfEJJ] 2023-09-22T13:04:35.244 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.242+0000 7f42f1ebc500 20 get_obj_state: rctx=0x5641384b4370 obj=check-bucket:_:ANUZGLNiZ5Cy6OE81kEqC-8tsfsfEJJ_e state=0x564138521e88 s->prefetch_data=0 2023-09-22T13:04:35.244 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.243+0000 7f42f1ebc500 10 manifest: total_size = 12 2023-09-22T13:04:35.244 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.243+0000 7f42f1ebc500 20 get_obj_state: setting s->obj_tag to 772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4689.3151756815556312425 2023-09-22T13:04:35.244 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.243+0000 7f42f1ebc500 20 get_obj_state: rctx=0x5641384b4370 obj=check-bucket:_:ANUZGLNiZ5Cy6OE81kEqC-8tsfsfEJJ_e state=0x564138521e88 s->prefetch_data=0 2023-09-22T13:04:35.244 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.243+0000 7f42f1ebc500 20 prepare_atomic_modification: state is not atomic. state=0x564138521e88 2023-09-22T13:04:35.244 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.243+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:35.248 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.248+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : miss 2023-09-22T13:04:35.248 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.248+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.249 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.248+0000 7f42f1ebc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.249 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.248+0000 7f42f1ebc500 10 cache put: name=default.rgw.log++bucket.sync-source-hints.check-bucket info.flags=0x0 2023-09-22T13:04:35.249 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.248+0000 7f42f1ebc500 10 adding default.rgw.log++bucket.sync-source-hints.check-bucket to cache LRU end 2023-09-22T13:04:35.249 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.248+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : miss 2023-09-22T13:04:35.249 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.248+0000 7f42f1ebc500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.249 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.249+0000 7f42f1ebc500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.249 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.249+0000 7f42f1ebc500 10 cache put: name=default.rgw.log++bucket.sync-target-hints.check-bucket info.flags=0x0 2023-09-22T13:04:35.250 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.249+0000 7f42f1ebc500 10 adding default.rgw.log++bucket.sync-target-hints.check-bucket to cache LRU end 2023-09-22T13:04:35.250 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.249+0000 7f42f1ebc500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:35.250 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.249+0000 7f42f1ebc500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:35.250 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.249+0000 7f42f1ebc500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:35.254 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.253+0000 7f42f1ebc500 20 get_obj_state: rctx=0x5641384b4370 obj=check-bucket:e state=0x5641384db2b8 s->prefetch_data=0 2023-09-22T13:04:35.254 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.253+0000 7f42f1ebc500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=ejvwkqja5a1thiwjjjy7wxf382n40id5 2023-09-22T13:04:35.263 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.262+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:35.264 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.263+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.1 2023-09-22T13:04:35.264 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.263+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.264 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.263+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.265 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.263+0000 7f42f1ebc500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:35.265 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.263+0000 7f42f1ebc500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:35.265 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.263+0000 7f42f1ebc500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:35.265 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.263+0000 7f42f1ebc500 1 NOTICE: finished shard 1 (1 entries removed) 2023-09-22T13:04:35.265 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.263+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:35.265 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.264+0000 7f42f1ebc500 20 get_obj_state: rctx=0x5641384d8570 obj=check-bucket:d state=0x5641384db2b8 s->prefetch_data=0 2023-09-22T13:04:35.265 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.264+0000 7f42f1ebc500 20 get_obj_state: setting s->obj_tag to g98lt8kr9e5xz4ycyta2wym43vb16d7b 2023-09-22T13:04:35.266 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.264+0000 7f42f1ebc500 20 get_obj_state_impl: setting s->olh_tag to il9hapwfpywgvryqyd05wefx1cswdht1 2023-09-22T13:04:35.266 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.264+0000 7f42f1ebc500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=il9hapwfpywgvryqyd05wefx1cswdht1 2023-09-22T13:04:35.266 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.265+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:35.267 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.267+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:35.267 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.267+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:35.268 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.267+0000 7f42f1ebc500 20 olh_log_entry: epoch=3 op=3 key=d[CWTvnda-Qyzn4tjnxTu4.dZKt56nOHu] 2023-09-22T13:04:35.268 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.267+0000 7f42f1ebc500 20 get_obj_state: rctx=0x5641384d8570 obj=check-bucket:_:CWTvnda-Qyzn4tjnxTu4.dZKt56nOHu_d state=0x56413851be28 s->prefetch_data=0 2023-09-22T13:04:35.268 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.268+0000 7f42f1ebc500 10 manifest: total_size = 11 2023-09-22T13:04:35.268 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.268+0000 7f42f1ebc500 20 get_obj_state: setting s->obj_tag to 772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4689.12253182639745054753 2023-09-22T13:04:35.268 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.268+0000 7f42f1ebc500 20 get_obj_state: rctx=0x5641384d8570 obj=check-bucket:_:CWTvnda-Qyzn4tjnxTu4.dZKt56nOHu_d state=0x56413851be28 s->prefetch_data=0 2023-09-22T13:04:35.269 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.268+0000 7f42f1ebc500 20 prepare_atomic_modification: state is not atomic. state=0x56413851be28 2023-09-22T13:04:35.269 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.268+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:35.272 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.271+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.272 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.271+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.272 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.271+0000 7f42f1ebc500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:35.272 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.271+0000 7f42f1ebc500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:35.272 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.271+0000 7f42f1ebc500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:35.274 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.273+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:35.274 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.273+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:35.275 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.274+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.5 2023-09-22T13:04:35.275 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.274+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.274+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.274+0000 7f42f1ebc500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:35.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.274+0000 7f42f1ebc500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:35.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.274+0000 7f42f1ebc500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:35.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.274+0000 7f42f1ebc500 1 NOTICE: finished shard 5 (1 entries removed) 2023-09-22T13:04:35.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.274+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:35.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.275+0000 7f42f1ebc500 20 get_obj_state: rctx=0x5641384fad70 obj=check-bucket:c state=0x5641384db2b8 s->prefetch_data=0 2023-09-22T13:04:35.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.275+0000 7f42f1ebc500 20 get_obj_state: setting s->obj_tag to 6ngoy6m5btz74xmrgzvqdgz2aqey4zm1 2023-09-22T13:04:35.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.275+0000 7f42f1ebc500 20 get_obj_state_impl: setting s->olh_tag to n5lnen3kib4vkg0fuoxla41z21qx6lpt 2023-09-22T13:04:35.276 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.275+0000 7f42f1ebc500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=n5lnen3kib4vkg0fuoxla41z21qx6lpt 2023-09-22T13:04:35.277 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.277+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:35.279 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.278+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:35.279 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.278+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:35.279 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.279+0000 7f42f1ebc500 20 olh_log_entry: epoch=3 op=3 key=c[DwcphBqm7XBQ9QUA1PSkZVyWuktKzYJ] 2023-09-22T13:04:35.279 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.279+0000 7f42f1ebc500 20 get_obj_state: rctx=0x5641384fad70 obj=check-bucket:_:DwcphBqm7XBQ9QUA1PSkZVyWuktKzYJ_c state=0x56413851be28 s->prefetch_data=0 2023-09-22T13:04:35.280 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.279+0000 7f42f1ebc500 10 manifest: total_size = 9 2023-09-22T13:04:35.280 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.279+0000 7f42f1ebc500 20 get_obj_state: setting s->obj_tag to 772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4689.12129622947710950758 2023-09-22T13:04:35.280 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.279+0000 7f42f1ebc500 20 get_obj_state: rctx=0x5641384fad70 obj=check-bucket:_:DwcphBqm7XBQ9QUA1PSkZVyWuktKzYJ_c state=0x56413851be28 s->prefetch_data=0 2023-09-22T13:04:35.280 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.279+0000 7f42f1ebc500 20 prepare_atomic_modification: state is not atomic. state=0x56413851be28 2023-09-22T13:04:35.280 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.279+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:35.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.282+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.282+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.282+0000 7f42f1ebc500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:35.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.282+0000 7f42f1ebc500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:35.283 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.282+0000 7f42f1ebc500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:35.286 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.285+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:35.286 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.285+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:35.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.8 2023-09-22T13:04:35.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:35.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:35.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:35.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 1 NOTICE: finished shard 8 (1 entries removed) 2023-09-22T13:04:35.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:35.288 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 20 get_obj_state: rctx=0x56413851ae70 obj=check-bucket:f state=0x56413851be28 s->prefetch_data=0 2023-09-22T13:04:35.289 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 20 get_obj_state: setting s->obj_tag to rcpwrusob86158lwt3xja3324v9n05ya 2023-09-22T13:04:35.289 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 20 get_obj_state_impl: setting s->olh_tag to oh2gh4r2w9svnrlkyrrrtpvre8l0dwz6 2023-09-22T13:04:35.289 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.287+0000 7f42f1ebc500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=oh2gh4r2w9svnrlkyrrrtpvre8l0dwz6 2023-09-22T13:04:35.289 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.289+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:35.290 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.290+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:35.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.290+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:35.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.290+0000 7f42f1ebc500 20 olh_log_entry: epoch=2 op=2 key=f[] 2023-09-22T13:04:35.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.290+0000 7f42f1ebc500 20 olh_log_entry: epoch=2 op=3 key=f[7LN.ZddNKvWPX2kEWdGM.t5kwLgGoZA] 2023-09-22T13:04:35.291 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.290+0000 7f42f1ebc500 20 get_obj_state: rctx=0x56413851ae70 obj=check-bucket:_:7LN.ZddNKvWPX2kEWdGM.t5kwLgGoZA_f state=0x5641384db2b8 s->prefetch_data=0 2023-09-22T13:04:35.292 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.291+0000 7f42f1ebc500 10 manifest: total_size = 12 2023-09-22T13:04:35.292 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.291+0000 7f42f1ebc500 20 get_obj_state: setting s->obj_tag to 772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4689.4913353458058050025 2023-09-22T13:04:35.292 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.291+0000 7f42f1ebc500 20 get_obj_state: rctx=0x56413851ae70 obj=check-bucket:_:7LN.ZddNKvWPX2kEWdGM.t5kwLgGoZA_f state=0x5641384db2b8 s->prefetch_data=0 2023-09-22T13:04:35.292 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.291+0000 7f42f1ebc500 20 prepare_atomic_modification: state is not atomic. state=0x5641384db2b8 2023-09-22T13:04:35.292 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.291+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:35.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.293+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.293+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.293+0000 7f42f1ebc500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:35.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.293+0000 7f42f1ebc500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:35.294 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.293+0000 7f42f1ebc500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:35.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.295+0000 7f42f1ebc500 20 get_obj_state: rctx=0x56413851ae70 obj=check-bucket:f state=0x56413851be28 s->prefetch_data=0 2023-09-22T13:04:35.295 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.295+0000 7f42f1ebc500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=oh2gh4r2w9svnrlkyrrrtpvre8l0dwz6 2023-09-22T13:04:35.298 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.298+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:35.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.299+0000 7f42f1ebc500 20 bucket index object: default.rgw.buckets.index:.dir.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1.10 2023-09-22T13:04:35.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.299+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.299+0000 7f42f1ebc500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:04:35.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.299+0000 7f42f1ebc500 10 chain_cache_entry: cache_locator= 2023-09-22T13:04:35.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.299+0000 7f42f1ebc500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:04:35.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.299+0000 7f42f1ebc500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:04:35.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.299+0000 7f42f1ebc500 1 NOTICE: finished shard 10 (1 entries removed) 2023-09-22T13:04:35.300 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.299+0000 7f42f1ebc500 1 NOTICE: finished all shards (4 entries removed) 2023-09-22T13:04:35.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.301+0000 7f42f1ebc500 20 remove_watcher() i=0 2023-09-22T13:04:35.301 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.301+0000 7f42f1ebc500 2 removed watcher, disabling cache 2023-09-22T13:04:35.302 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.302+0000 7f42f1ebc500 20 remove_watcher() i=1 2023-09-22T13:04:35.305 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.305+0000 7f42f1ebc500 20 remove_watcher() i=2 2023-09-22T13:04:35.307 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.307+0000 7f42f1ebc500 20 remove_watcher() i=3 2023-09-22T13:04:35.309 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.308+0000 7f42f1ebc500 20 remove_watcher() i=4 2023-09-22T13:04:35.310 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.309+0000 7f42f1ebc500 20 remove_watcher() i=5 2023-09-22T13:04:35.311 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.310+0000 7f42f1ebc500 20 remove_watcher() i=6 2023-09-22T13:04:35.312 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.312+0000 7f42f1ebc500 20 remove_watcher() i=7 2023-09-22T13:04:35.313 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.313+0000 7f42ae7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:35.495 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:35.495 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:35.495 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:35.495 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:35.495 INFO:tasks.workunit.client.0.smithi170.stderr:radosgw-admin bucket stats --bucket check-bucket 2023-09-22T13:04:35.517 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:04:35.517 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:04:35.525 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.524+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.527 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.526+0000 7f8539bf2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.527 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.526+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.527+0000 7f8539bf2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.527+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.527+0000 7f8539bf2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.527+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.528 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.528+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:35.539 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.538+0000 7f8539bf2500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:04:35.539 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.538+0000 7f8539bf2500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:04:35.539 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.538+0000 7f8539bf2500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:04:35.539 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.538+0000 7f8539bf2500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:04:35.539 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.538+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.539 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.538+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:35.539 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.538+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.541 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.539+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:35.541 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.539+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.541 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.539+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:35.541 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.539+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.539+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.539+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 20 zonegroup default 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.542 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.543 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 10 Cannot find current period zone using local zone 2023-09-22T13:04:35.543 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.540+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.543 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.541+0000 7f8539bf2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.543 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.541+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.543 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.541+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:04:35.543 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.541+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.544 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.541+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:04:35.544 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.541+0000 7f8539bf2500 20 zone default found 2023-09-22T13:04:35.544 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.541+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.544 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.541+0000 7f8539bf2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.544 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.541+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.544 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.542+0000 7f8539bf2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:04:35.544 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.542+0000 7f8539bf2500 20 started sync module instance, tier type = 2023-09-22T13:04:35.544 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.542+0000 7f8539bf2500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:04:35.552 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.552+0000 7f8539bf2500 20 add_watcher() i=0 2023-09-22T13:04:35.553 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.552+0000 7f8539bf2500 20 add_watcher() i=1 2023-09-22T13:04:35.553 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.552+0000 7f8539bf2500 20 add_watcher() i=2 2023-09-22T13:04:35.553 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.552+0000 7f8539bf2500 20 add_watcher() i=3 2023-09-22T13:04:35.553 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.552+0000 7f8539bf2500 20 add_watcher() i=4 2023-09-22T13:04:35.553 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.552+0000 7f8539bf2500 20 add_watcher() i=5 2023-09-22T13:04:35.553 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.552+0000 7f8539bf2500 20 add_watcher() i=6 2023-09-22T13:04:35.553 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.552+0000 7f85248d3700 0 -- 172.21.15.170:0/1511600830 >> [v2:172.21.15.191:6808/144991,v1:172.21.15.191:6809/144991] conn(0x7f851c04d8c0 msgr2=0x7f851c04c970 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:04:35.555 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.554+0000 7f8539bf2500 20 add_watcher() i=7 2023-09-22T13:04:35.555 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.554+0000 7f8539bf2500 2 all 8 watchers are set, enabling cache 2023-09-22T13:04:35.557 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.557+0000 7f84f67e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:35.558 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.557+0000 7f8539bf2500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:04:35.558 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.557+0000 7f8539bf2500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:04:35.561 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.561+0000 7f85258d5700 0 -- 172.21.15.170:0/598858524 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x5611ed941090 msgr2=0x5611ed93ab20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:04:35.762 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.762+0000 7f85258d5700 0 --2- 172.21.15.170:0/598858524 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x5611ed941090 0x5611ed93ab20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:04:35.764 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.764+0000 7f8539bf2500 5 note: GC not initialized 2023-09-22T13:04:35.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.765+0000 7f84657fa700 20 reqs_thread_entry: start 2023-09-22T13:04:35.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.765+0000 7f845effd700 20 reqs_thread_entry: start 2023-09-22T13:04:35.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.765+0000 7f8539bf2500 20 init_complete bucket index max shards: 11 2023-09-22T13:04:35.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.765+0000 7f8539bf2500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:04:35.766 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.765+0000 7f845d7fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:05:05 2023 2023-09-22T13:04:35.767 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:04:35.767 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.765+0000 7f845cff9700 20 reqs_thread_entry: start 2023-09-22T13:04:35.767 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.765+0000 7f8539bf2500 20 get_system_obj_state: rctx=0x7fffc181cc98 obj=default.rgw.meta:root:check-bucket state=0x5611ed910190 s->prefetch_data=0 2023-09-22T13:04:35.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.766+0000 7f8539bf2500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:35.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.766+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.766+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:35.768 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.766+0000 7f8539bf2500 20 get_system_obj_state: rctx=0x7fffc181cd68 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5611ed910190 s->prefetch_data=0 2023-09-22T13:04:35.769 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.767+0000 7f8539bf2500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:35.769 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.767+0000 7f8539bf2500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:35.769 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.767+0000 7f8539bf2500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:35.769 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.767+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.769 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.767+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:35.769 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.767+0000 7f8539bf2500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:35.769 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.767+0000 7f8539bf2500 20 get_system_obj_state: rctx=0x7fffc181d060 obj=default.rgw.meta:root:check-bucket state=0x5611ed910190 s->prefetch_data=0 2023-09-22T13:04:35.769 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.767+0000 7f8539bf2500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:35.769 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.767+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.769 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.768+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:04:35.769 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.768+0000 7f8539bf2500 20 get_system_obj_state: rctx=0x7fffc181d060 obj=default.rgw.meta:root:.bucket.meta.check-bucket:772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.1 state=0x5611ed833b50 s->prefetch_data=0 2023-09-22T13:04:35.770 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.768+0000 7f8539bf2500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:04:35.770 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.768+0000 7f8539bf2500 20 Read xattr: user.rgw.acl 2023-09-22T13:04:35.770 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.768+0000 7f8539bf2500 20 Read xattr: user.rgw.idtag 2023-09-22T13:04:35.770 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.768+0000 7f8539bf2500 20 rados->read ofs=0 len=0 2023-09-22T13:04:35.770 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.768+0000 7f8539bf2500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:04:35.770 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.768+0000 7f8539bf2500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:04:35.771 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.771+0000 7f8539bf2500 20 remove_watcher() i=0 2023-09-22T13:04:35.771 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.771+0000 7f8539bf2500 2 removed watcher, disabling cache 2023-09-22T13:04:35.772 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.772+0000 7f8539bf2500 20 remove_watcher() i=1 2023-09-22T13:04:35.773 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.773+0000 7f8539bf2500 20 remove_watcher() i=2 2023-09-22T13:04:35.774 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.774+0000 7f8539bf2500 20 remove_watcher() i=3 2023-09-22T13:04:35.775 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.775+0000 7f8539bf2500 20 remove_watcher() i=4 2023-09-22T13:04:35.776 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.776+0000 7f8539bf2500 20 remove_watcher() i=5 2023-09-22T13:04:35.777 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.777+0000 7f8539bf2500 20 remove_watcher() i=6 2023-09-22T13:04:35.779 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.778+0000 7f8539bf2500 20 remove_watcher() i=7 2023-09-22T13:04:35.779 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:04:35.779+0000 7f84f67e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:04:35.784 INFO:tasks.workunit.client.0.smithi170.stderr:{'rgw.main': {'size': 0, 'size_actual': 0, 'size_utilized': 0, 'size_kb': 0, 'size_kb_actual': 0, 'size_kb_utilized': 0, 'num_objects': 0}} 2023-09-22T13:04:35.785 INFO:tasks.workunit.client.0.smithi170.stderr:Deleting bucket check-bucket 2023-09-22T13:04:35.825 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:35.826 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:35.826 INFO:tasks.workunit.client.0.smithi170.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/urllib3/connectionpool.py:1063: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings 2023-09-22T13:04:35.826 INFO:tasks.workunit.client.0.smithi170.stderr: InsecureRequestWarning, 2023-09-22T13:04:35.826 INFO:tasks.workunit.client.0.smithi170.stderr:Completed bucket check tests 2023-09-22T13:04:35.826 INFO:tasks.workunit.client.0.smithi170.stdout:Could not connect to the endpoint URL: "http://localhost:80/" 2023-09-22T13:04:35.826 INFO:tasks.workunit.client.0.smithi170.stdout:Could not connect to the endpoint URL: "http://localhost:8000/" 2023-09-22T13:04:35.826 INFO:tasks.workunit.client.0.smithi170.stdout:connected to https://localhost:443 2023-09-22T13:04:35.861 INFO:tasks.workunit.client.0.smithi170.stderr:+ deactivate 2023-09-22T13:04:35.861 INFO:tasks.workunit.client.0.smithi170.stderr:+ '[' -n /home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin ']' 2023-09-22T13:04:35.861 INFO:tasks.workunit.client.0.smithi170.stderr:+ PATH=/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin 2023-09-22T13:04:35.861 INFO:tasks.workunit.client.0.smithi170.stderr:+ export PATH 2023-09-22T13:04:35.861 INFO:tasks.workunit.client.0.smithi170.stderr:+ unset _OLD_VIRTUAL_PATH 2023-09-22T13:04:35.861 INFO:tasks.workunit.client.0.smithi170.stderr:+ '[' -n '' ']' 2023-09-22T13:04:35.861 INFO:tasks.workunit.client.0.smithi170.stderr:+ '[' -n /usr/bin/bash -o -n '' ']' 2023-09-22T13:04:35.861 INFO:tasks.workunit.client.0.smithi170.stderr:+ hash -r 2023-09-22T13:04:35.861 INFO:tasks.workunit.client.0.smithi170.stderr:+ '[' -n '' ']' 2023-09-22T13:04:35.861 INFO:tasks.workunit.client.0.smithi170.stderr:+ unset VIRTUAL_ENV 2023-09-22T13:04:35.862 INFO:tasks.workunit.client.0.smithi170.stderr:+ '[' '!' '' = nondestructive ']' 2023-09-22T13:04:35.862 INFO:tasks.workunit.client.0.smithi170.stderr:+ unset -f deactivate 2023-09-22T13:04:35.862 INFO:tasks.workunit.client.0.smithi170.stderr:+ echo OK. 2023-09-22T13:04:35.862 INFO:tasks.workunit.client.0.smithi170.stdout:OK. 2023-09-22T13:04:35.863 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:04:35.863 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:04:35.930 INFO:tasks.workunit:Stopping ['rgw/run-bucket-check.sh'] on client.0... 2023-09-22T13:04:35.930 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-22T13:04:36.257 DEBUG:teuthology.parallel:result is None 2023-09-22T13:04:36.258 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:04:36.285 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:04:36.286 DEBUG:teuthology.orchestra.run.smithi170:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:04:36.347 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:04:36.347 INFO:teuthology.run_tasks:Running task workunit... 2023-09-22T13:04:36.357 INFO:tasks.workunit:Pulling workunits from ref 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:04:36.357 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-22T13:04:36.357 DEBUG:teuthology.orchestra.run.smithi170:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:04:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T13:04:36.406 INFO:teuthology.orchestra.run.smithi170.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-22T13:04:36.406 DEBUG:teuthology.orchestra.run.smithi170:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:04:36.464 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-22T13:04:36.465 DEBUG:teuthology.orchestra.run.smithi170:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-22T13:04:36.522 INFO:tasks.workunit:timeout=3h 2023-09-22T13:04:36.523 INFO:tasks.workunit:cleanup=True 2023-09-22T13:04:36.523 DEBUG:teuthology.orchestra.run.smithi170:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:04:36.581 INFO:tasks.workunit.client.0.smithi170.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-22T13:05:36.751 INFO:tasks.workunit.client.0.smithi170.stderr:Note: switching to '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'. 2023-09-22T13:05:36.752 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:05:36.752 INFO:tasks.workunit.client.0.smithi170.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2023-09-22T13:05:36.752 INFO:tasks.workunit.client.0.smithi170.stderr:changes and commit them, and you can discard any commits you make in this 2023-09-22T13:05:36.752 INFO:tasks.workunit.client.0.smithi170.stderr:state without impacting any branches by switching back to a branch. 2023-09-22T13:05:36.752 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:05:36.752 INFO:tasks.workunit.client.0.smithi170.stderr:If you want to create a new branch to retain commits you create, you may 2023-09-22T13:05:36.752 INFO:tasks.workunit.client.0.smithi170.stderr:do so (now or later) by using -c with the switch command. Example: 2023-09-22T13:05:36.753 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:05:36.753 INFO:tasks.workunit.client.0.smithi170.stderr: git switch -c 2023-09-22T13:05:36.753 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:05:36.753 INFO:tasks.workunit.client.0.smithi170.stderr:Or undo this operation with: 2023-09-22T13:05:36.753 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:05:36.753 INFO:tasks.workunit.client.0.smithi170.stderr: git switch - 2023-09-22T13:05:36.753 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:05:36.753 INFO:tasks.workunit.client.0.smithi170.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2023-09-22T13:05:36.754 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:05:36.754 INFO:tasks.workunit.client.0.smithi170.stderr:HEAD is now at 31c126b7c1c rgw: radosgw-admin bucket check should only print index entries with --check-objects flag 2023-09-22T13:05:36.757 DEBUG:teuthology.orchestra.run.smithi170:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-22T13:05:36.815 INFO:tasks.workunit.client.0.smithi170.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-22T13:05:36.817 INFO:tasks.workunit.client.0.smithi170.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:05:36.818 INFO:tasks.workunit.client.0.smithi170.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-22T13:05:36.864 INFO:tasks.workunit.client.0.smithi170.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-22T13:05:36.910 INFO:tasks.workunit.client.0.smithi170.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-22T13:05:36.948 INFO:tasks.workunit.client.0.smithi170.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:05:36.949 INFO:tasks.workunit.client.0.smithi170.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:05:36.950 INFO:tasks.workunit.client.0.smithi170.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-22T13:05:36.986 INFO:tasks.workunit.client.0.smithi170.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:05:36.990 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:05:36.990 DEBUG:teuthology.orchestra.run.smithi170:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-22T13:05:37.049 INFO:tasks.workunit:Running workunits matching cls/test_cls_lock.sh on client.0... 2023-09-22T13:05:37.050 INFO:tasks.workunit:Running workunit cls/test_cls_lock.sh... 2023-09-22T13:05:37.050 DEBUG:teuthology.orchestra.run.smithi170:workunit test cls/test_cls_lock.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_lock.sh 2023-09-22T13:05:37.121 INFO:tasks.workunit.client.0.smithi170.stdout:Running main() from gmock_main.cc 2023-09-22T13:05:37.121 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 11 tests from 1 test suite. 2023-09-22T13:05:37.121 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:05:37.121 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 11 tests from ClsLock 2023-09-22T13:05:37.121 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] ClsLock.TestMultiLocking 2023-09-22T13:05:38.603 INFO:tasks.workunit.client.0.smithi170.stdout:lock: mylock 2023-09-22T13:05:38.604 INFO:tasks.workunit.client.0.smithi170.stdout: lock_type: exclusive 2023-09-22T13:05:38.604 INFO:tasks.workunit.client.0.smithi170.stdout: tag: 2023-09-22T13:05:38.604 INFO:tasks.workunit.client.0.smithi170.stdout: lockers: 2023-09-22T13:05:38.604 INFO:tasks.workunit.client.0.smithi170.stdout: client.5133 expiration=2023-09-22T13:07:38.600817+0000 addr=v1:172.21.15.170:0/3991571466 cookie= 2023-09-22T13:05:38.607 INFO:tasks.workunit.client.0.smithi170.stdout:lock: mylock 2023-09-22T13:05:38.607 INFO:tasks.workunit.client.0.smithi170.stdout: lock_type: shared 2023-09-22T13:05:38.607 INFO:tasks.workunit.client.0.smithi170.stdout: tag: 2023-09-22T13:05:38.607 INFO:tasks.workunit.client.0.smithi170.stdout: lockers: 2023-09-22T13:05:38.608 INFO:tasks.workunit.client.0.smithi170.stdout: client.4934 expiration=0.000000 addr=v1:172.21.15.170:0/2375623887 cookie= 2023-09-22T13:05:38.608 INFO:tasks.workunit.client.0.smithi170.stdout: client.5133 expiration=2023-09-22T13:07:38.606716+0000 addr=v1:172.21.15.170:0/3991571466 cookie= 2023-09-22T13:05:38.608 INFO:tasks.workunit.client.0.smithi170.stdout:lock: mylock 2023-09-22T13:05:38.608 INFO:tasks.workunit.client.0.smithi170.stdout: lock_type: shared 2023-09-22T13:05:38.608 INFO:tasks.workunit.client.0.smithi170.stdout: tag: 2023-09-22T13:05:38.609 INFO:tasks.workunit.client.0.smithi170.stdout: lockers: 2023-09-22T13:05:38.609 INFO:tasks.workunit.client.0.smithi170.stdout: client.4934 expiration=0.000000 addr=v1:172.21.15.170:0/2375623887 cookie= 2023-09-22T13:05:39.574 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] ClsLock.TestMultiLocking (2453 ms) 2023-09-22T13:05:39.574 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] ClsLock.TestMeta 2023-09-22T13:05:41.632 INFO:tasks.workunit.client.0.smithi170.stdout:lock: mylock 2023-09-22T13:05:41.632 INFO:tasks.workunit.client.0.smithi170.stdout: lock_type: shared 2023-09-22T13:05:41.632 INFO:tasks.workunit.client.0.smithi170.stdout: tag: 2023-09-22T13:05:41.633 INFO:tasks.workunit.client.0.smithi170.stdout: lockers: 2023-09-22T13:05:41.633 INFO:tasks.workunit.client.0.smithi170.stdout: client.4943 expiration=0.000000 addr=v1:172.21.15.170:0/1050817972 cookie= 2023-09-22T13:05:41.634 INFO:tasks.workunit.client.0.smithi170.stdout:lock: mylock 2023-09-22T13:05:41.634 INFO:tasks.workunit.client.0.smithi170.stdout: lock_type: exclusive 2023-09-22T13:05:41.634 INFO:tasks.workunit.client.0.smithi170.stdout: tag: new_tag 2023-09-22T13:05:41.635 INFO:tasks.workunit.client.0.smithi170.stdout: lockers: 2023-09-22T13:05:41.635 INFO:tasks.workunit.client.0.smithi170.stdout: client.5142 expiration=0.000000 addr=v1:172.21.15.170:0/3569585313 cookie= 2023-09-22T13:05:42.601 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] ClsLock.TestMeta (3027 ms) 2023-09-22T13:05:42.601 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] ClsLock.TestCookie 2023-09-22T13:05:44.646 INFO:tasks.workunit.client.0.smithi170.stdout:lock: mylock 2023-09-22T13:05:44.646 INFO:tasks.workunit.client.0.smithi170.stdout: lock_type: exclusive 2023-09-22T13:05:44.646 INFO:tasks.workunit.client.0.smithi170.stdout: tag: 2023-09-22T13:05:44.646 INFO:tasks.workunit.client.0.smithi170.stdout: lockers: 2023-09-22T13:05:44.649 INFO:tasks.workunit.client.0.smithi170.stdout:lock: mylock 2023-09-22T13:05:44.649 INFO:tasks.workunit.client.0.smithi170.stdout: lock_type: shared 2023-09-22T13:05:44.650 INFO:tasks.workunit.client.0.smithi170.stdout: tag: 2023-09-22T13:05:44.650 INFO:tasks.workunit.client.0.smithi170.stdout: lockers: 2023-09-22T13:05:44.650 INFO:tasks.workunit.client.0.smithi170.stdout: client.4946 expiration=0.000000 addr=v1:172.21.15.170:0/1952266309 cookie= 2023-09-22T13:05:44.650 INFO:tasks.workunit.client.0.smithi170.stdout: client.4946 expiration=0.000000 addr=v1:172.21.15.170:0/1952266309 cookie=new cookie 2023-09-22T13:05:45.632 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] ClsLock.TestCookie (3029 ms) 2023-09-22T13:05:45.633 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] ClsLock.TestMultipleLocks 2023-09-22T13:05:48.652 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] ClsLock.TestMultipleLocks (3023 ms) 2023-09-22T13:05:48.652 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] ClsLock.TestLockDuration 2023-09-22T13:05:56.679 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] ClsLock.TestLockDuration (8026 ms) 2023-09-22T13:05:56.679 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] ClsLock.TestAssertLocked 2023-09-22T13:05:59.694 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] ClsLock.TestAssertLocked (3015 ms) 2023-09-22T13:05:59.694 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] ClsLock.TestSetCookie 2023-09-22T13:06:02.703 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] ClsLock.TestSetCookie (3009 ms) 2023-09-22T13:06:02.703 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] ClsLock.TestRenew 2023-09-22T13:06:23.763 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] ClsLock.TestRenew (21060 ms) 2023-09-22T13:06:23.763 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralBasic 2023-09-22T13:06:30.534 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] ClsLock.TestExclusiveEphemeralBasic (6772 ms) 2023-09-22T13:06:30.534 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealEphemeral 2023-09-22T13:06:38.563 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealEphemeral (8026 ms) 2023-09-22T13:06:38.564 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealExclusive 2023-09-22T13:06:46.588 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealExclusive (8027 ms) 2023-09-22T13:06:46.588 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 11 tests from ClsLock (69467 ms total) 2023-09-22T13:06:46.588 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:06:46.589 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:06:46.589 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 11 tests from 1 test suite ran. (69467 ms total) 2023-09-22T13:06:46.589 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 11 tests. 2023-09-22T13:06:46.591 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:06:46.591 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:06:46.664 INFO:tasks.workunit:Running workunits matching cls/test_cls_log.sh on client.0... 2023-09-22T13:06:46.665 INFO:tasks.workunit:Running workunit cls/test_cls_log.sh... 2023-09-22T13:06:46.666 DEBUG:teuthology.orchestra.run.smithi170:workunit test cls/test_cls_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_log.sh 2023-09-22T13:06:46.742 INFO:tasks.workunit.client.0.smithi170.stdout:Running main() from gmock_main.cc 2023-09-22T13:06:46.742 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 4 tests from 1 test suite. 2023-09-22T13:06:46.742 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:06:46.742 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 4 tests from cls_log 2023-09-22T13:06:46.742 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_log.test_log_add_same_time 2023-09-22T13:06:49.603 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_log.test_log_add_same_time (2861 ms) 2023-09-22T13:06:49.603 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_log.test_log_add_different_time 2023-09-22T13:06:52.618 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_log.test_log_add_different_time (3015 ms) 2023-09-22T13:06:52.618 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_log.trim_by_time 2023-09-22T13:06:55.636 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_log.trim_by_time (3015 ms) 2023-09-22T13:06:55.637 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_log.trim_by_marker 2023-09-22T13:06:58.653 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_log.trim_by_marker (3020 ms) 2023-09-22T13:06:58.653 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 4 tests from cls_log (11912 ms total) 2023-09-22T13:06:58.654 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:06:58.654 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:06:58.654 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 4 tests from 1 test suite ran. (11912 ms total) 2023-09-22T13:06:58.654 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 4 tests. 2023-09-22T13:06:58.655 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:06:58.656 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:06:58.726 INFO:tasks.workunit:Running workunits matching cls/test_cls_refcount.sh on client.0... 2023-09-22T13:06:58.727 INFO:tasks.workunit:Running workunit cls/test_cls_refcount.sh... 2023-09-22T13:06:58.728 DEBUG:teuthology.orchestra.run.smithi170:workunit test cls/test_cls_refcount.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_refcount.sh 2023-09-22T13:06:58.800 INFO:tasks.workunit.client.0.smithi170.stdout:Running main() from gmock_main.cc 2023-09-22T13:06:58.800 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 10 tests from 1 test suite. 2023-09-22T13:06:58.800 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:06:58.801 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 10 tests from cls_rgw 2023-09-22T13:06:58.801 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.test_implicit 2023-09-22T13:07:01.662 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.test_implicit (2859 ms) 2023-09-22T13:07:01.662 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.test_implicit_idempotent 2023-09-22T13:07:04.688 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.test_implicit_idempotent (3027 ms) 2023-09-22T13:07:04.689 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.test_put_snap 2023-09-22T13:07:13.720 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.test_put_snap (9033 ms) 2023-09-22T13:07:13.720 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.test_explicit 2023-09-22T13:07:16.746 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.test_explicit (3025 ms) 2023-09-22T13:07:16.746 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.set 2023-09-22T13:07:19.755 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.set (3010 ms) 2023-09-22T13:07:19.756 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.test_implicit_ec 2023-09-22T13:07:26.792 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.test_implicit_ec (7037 ms) 2023-09-22T13:07:26.792 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.test_implicit_idempotent_ec 2023-09-22T13:07:33.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.test_implicit_idempotent_ec (7026 ms) 2023-09-22T13:07:33.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.test_put_snap_ec 2023-09-22T13:07:46.872 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.test_put_snap_ec (13054 ms) 2023-09-22T13:07:46.872 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.test_explicit_ec 2023-09-22T13:07:53.907 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.test_explicit_ec (7035 ms) 2023-09-22T13:07:53.908 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.set_ec 2023-09-22T13:08:00.929 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.set_ec (7021 ms) 2023-09-22T13:08:00.929 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 10 tests from cls_rgw (62128 ms total) 2023-09-22T13:08:00.929 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:08:00.929 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:08:00.930 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 10 tests from 1 test suite ran. (62128 ms total) 2023-09-22T13:08:00.930 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 10 tests. 2023-09-22T13:08:00.931 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:08:00.931 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:08:01.004 INFO:tasks.workunit:Running workunits matching cls/test_cls_rgw.sh on client.0... 2023-09-22T13:08:01.006 INFO:tasks.workunit:Running workunit cls/test_cls_rgw.sh... 2023-09-22T13:08:01.006 DEBUG:teuthology.orchestra.run.smithi170:workunit test cls/test_cls_rgw.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rgw.sh 2023-09-22T13:08:01.079 INFO:tasks.workunit.client.0.smithi170.stdout:Running main() from gmock_main.cc 2023-09-22T13:08:01.080 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 16 tests from 1 test suite. 2023-09-22T13:08:01.080 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:08:01.080 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 16 tests from cls_rgw 2023-09-22T13:08:02.945 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.index_basic 2023-09-22T13:08:03.007 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.index_basic (63 ms) 2023-09-22T13:08:03.008 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.index_multiple_obj_writers 2023-09-22T13:08:03.042 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.index_multiple_obj_writers (34 ms) 2023-09-22T13:08:03.042 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.index_remove_object 2023-09-22T13:08:03.076 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.index_remove_object (35 ms) 2023-09-22T13:08:03.077 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.index_suggest 2023-09-22T13:08:04.687 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.index_suggest (1611 ms) 2023-09-22T13:08:04.688 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.index_suggest_complete 2023-09-22T13:08:04.693 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.index_suggest_complete (6 ms) 2023-09-22T13:08:04.693 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.index_list 2023-09-22T13:08:04.718 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.index_list (25 ms) 2023-09-22T13:08:04.718 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.index_list_delimited 2023-09-22T13:08:29.957 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.index_list_delimited (25238 ms) 2023-09-22T13:08:29.957 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.bi_list 2023-09-22T13:08:30.102 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.bi_list (146 ms) 2023-09-22T13:08:30.103 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.gc_set 2023-09-22T13:08:30.117 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.gc_set (15 ms) 2023-09-22T13:08:30.117 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.gc_list 2023-09-22T13:08:30.130 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.gc_list (13 ms) 2023-09-22T13:08:30.130 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.gc_defer 2023-09-22T13:08:37.598 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.gc_defer (7467 ms) 2023-09-22T13:08:37.598 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.usage_basic 2023-09-22T13:08:37.691 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.usage_basic (93 ms) 2023-09-22T13:08:37.692 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.usage_clear_no_obj 2023-09-22T13:08:37.692 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.usage_clear_no_obj (2 ms) 2023-09-22T13:08:37.692 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.usage_clear 2023-09-22T13:08:37.803 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.usage_clear (111 ms) 2023-09-22T13:08:37.803 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.bi_log_trim 2023-09-22T13:08:38.076 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.bi_log_trim (272 ms) 2023-09-22T13:08:38.076 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] cls_rgw.index_racing_removes 2023-09-22T13:08:38.098 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] cls_rgw.index_racing_removes (22 ms) 2023-09-22T13:08:38.607 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 16 tests from cls_rgw (35153 ms total) 2023-09-22T13:08:38.607 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:08:38.607 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:08:38.607 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 16 tests from 1 test suite ran. (37527 ms total) 2023-09-22T13:08:38.607 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 16 tests. 2023-09-22T13:08:38.610 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:08:38.610 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:08:38.681 INFO:tasks.workunit:Running workunits matching cls/test_cls_cmpomap.sh on client.0... 2023-09-22T13:08:38.682 INFO:tasks.workunit:Running workunit cls/test_cls_cmpomap.sh... 2023-09-22T13:08:38.683 DEBUG:teuthology.orchestra.run.smithi170:workunit test cls/test_cls_cmpomap.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cmpomap.sh 2023-09-22T13:08:38.756 INFO:tasks.workunit.client.0.smithi170.stdout:Running main() from gmock_main.cc 2023-09-22T13:08:38.756 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 31 tests from 1 test suite. 2023-09-22T13:08:38.756 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:08:40.610 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 31 tests from CmpOmap 2023-09-22T13:08:40.610 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str 2023-09-22T13:08:40.634 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str (25 ms) 2023-09-22T13:08:40.635 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str_default 2023-09-22T13:08:40.638 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str_default (4 ms) 2023-09-22T13:08:40.639 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64 2023-09-22T13:08:40.642 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64 (3 ms) 2023-09-22T13:08:40.642 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64_default 2023-09-22T13:08:40.646 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64_default (4 ms) 2023-09-22T13:08:40.646 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_str 2023-09-22T13:08:40.653 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_str (8 ms) 2023-09-22T13:08:40.653 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_u64 2023-09-22T13:08:40.660 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_u64 (7 ms) 2023-09-22T13:08:40.660 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_input 2023-09-22T13:08:40.664 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_input (4 ms) 2023-09-22T13:08:40.665 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_default 2023-09-22T13:08:40.668 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_default (4 ms) 2023-09-22T13:08:40.668 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_u64_empty_default 2023-09-22T13:08:40.672 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_u64_empty_default (3 ms) 2023-09-22T13:08:40.672 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_value 2023-09-22T13:08:40.677 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_value (5 ms) 2023-09-22T13:08:40.677 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_at_max_keys 2023-09-22T13:08:40.677 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_at_max_keys (1 ms) 2023-09-22T13:08:40.678 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_vals_over_max_keys 2023-09-22T13:08:40.678 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_vals_over_max_keys (1 ms) 2023-09-22T13:08:40.679 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str 2023-09-22T13:08:40.686 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str (7 ms) 2023-09-22T13:08:40.686 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str_default 2023-09-22T13:08:40.693 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str_default (8 ms) 2023-09-22T13:08:40.694 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64 2023-09-22T13:08:40.700 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64 (6 ms) 2023-09-22T13:08:40.700 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64_default 2023-09-22T13:08:40.707 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64_default (7 ms) 2023-09-22T13:08:40.707 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_set_vals_str 2023-09-22T13:08:40.715 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_set_vals_str (9 ms) 2023-09-22T13:08:40.715 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64 2023-09-22T13:08:40.726 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_set_vals_u64 (11 ms) 2023-09-22T13:08:40.726 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_einval 2023-09-22T13:08:40.730 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_einval (3 ms) 2023-09-22T13:08:40.730 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_eio 2023-09-22T13:08:40.733 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_eio (4 ms) 2023-09-22T13:08:40.733 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_set_vals_at_max_keys 2023-09-22T13:08:40.734 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_set_vals_at_max_keys (1 ms) 2023-09-22T13:08:40.734 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_set_vals_over_max_keys 2023-09-22T13:08:40.735 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_set_vals_over_max_keys (0 ms) 2023-09-22T13:08:40.735 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_str 2023-09-22T13:08:40.744 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_str (10 ms) 2023-09-22T13:08:40.744 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_u64 2023-09-22T13:08:40.750 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_u64 (6 ms) 2023-09-22T13:08:40.750 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_rm_keys_str 2023-09-22T13:08:40.763 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_rm_keys_str (13 ms) 2023-09-22T13:08:40.763 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64 2023-09-22T13:08:40.771 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64 (8 ms) 2023-09-22T13:08:40.771 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_einval 2023-09-22T13:08:40.773 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_einval (1 ms) 2023-09-22T13:08:40.773 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_eio 2023-09-22T13:08:40.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_eio (3 ms) 2023-09-22T13:08:40.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_rm_keys_at_max_keys 2023-09-22T13:08:40.776 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_rm_keys_at_max_keys (1 ms) 2023-09-22T13:08:40.776 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_rm_keys_over_max_keys 2023-09-22T13:08:40.776 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_rm_keys_over_max_keys (0 ms) 2023-09-22T13:08:40.777 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_empty 2023-09-22T13:08:40.787 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_empty (11 ms) 2023-09-22T13:08:40.788 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 31 tests from CmpOmap (178 ms total) 2023-09-22T13:08:40.788 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:08:40.788 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:08:41.625 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 31 tests from 1 test suite ran. (2868 ms total) 2023-09-22T13:08:41.625 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 31 tests. 2023-09-22T13:08:41.627 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:08:41.627 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:08:41.698 INFO:tasks.workunit:Running workunits matching cls/test_cls_2pc_queue.sh on client.0... 2023-09-22T13:08:41.699 INFO:tasks.workunit:Running workunit cls/test_cls_2pc_queue.sh... 2023-09-22T13:08:41.699 DEBUG:teuthology.orchestra.run.smithi170:workunit test cls/test_cls_2pc_queue.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh 2023-09-22T13:08:41.770 INFO:tasks.workunit.client.0.smithi170.stdout:Running main() from gmock_main.cc 2023-09-22T13:08:41.770 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 22 tests from 1 test suite. 2023-09-22T13:08:41.770 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:08:41.770 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 22 tests from TestCls2PCQueue 2023-09-22T13:08:41.770 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.GetCapacity 2023-09-22T13:08:44.648 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.GetCapacity (2879 ms) 2023-09-22T13:08:44.649 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.AsyncGetCapacity 2023-09-22T13:08:47.667 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.AsyncGetCapacity (3019 ms) 2023-09-22T13:08:47.668 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.Reserve 2023-09-22T13:08:50.677 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.Reserve (3009 ms) 2023-09-22T13:08:50.678 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.AsyncReserve 2023-09-22T13:08:53.693 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.AsyncReserve (3016 ms) 2023-09-22T13:08:53.693 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.Commit 2023-09-22T13:08:56.706 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.Commit (3013 ms) 2023-09-22T13:08:56.706 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.Abort 2023-09-22T13:08:59.720 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.Abort (3014 ms) 2023-09-22T13:08:59.720 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.ReserveError 2023-09-22T13:09:02.744 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.ReserveError (3023 ms) 2023-09-22T13:09:02.745 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.CommitError 2023-09-22T13:09:05.765 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.CommitError (3020 ms) 2023-09-22T13:09:05.765 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.AbortError 2023-09-22T13:09:08.779 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.AbortError (3016 ms) 2023-09-22T13:09:08.779 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.MultiReserve 2023-09-22T13:09:11.790 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.MultiReserve (3010 ms) 2023-09-22T13:09:11.790 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.MultiCommit 2023-09-22T13:09:14.817 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.MultiCommit (3027 ms) 2023-09-22T13:09:14.817 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.MultiAbort 2023-09-22T13:09:17.822 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.MultiAbort (3006 ms) 2023-09-22T13:09:17.822 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.ReserveCommit 2023-09-22T13:09:20.844 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.ReserveCommit (3021 ms) 2023-09-22T13:09:20.844 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.ReserveAbort 2023-09-22T13:09:23.855 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.ReserveAbort (3011 ms) 2023-09-22T13:09:23.855 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.ManualCleanup 2023-09-22T13:09:26.878 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.ManualCleanup (3022 ms) 2023-09-22T13:09:26.879 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.Cleanup 2023-09-22T13:09:29.888 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.Cleanup (3012 ms) 2023-09-22T13:09:29.888 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.MultiProducer 2023-09-22T13:09:43.572 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.MultiProducer (13683 ms) 2023-09-22T13:09:43.572 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.AsyncConsumer 2023-09-22T13:09:47.602 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.AsyncConsumer (4030 ms) 2023-09-22T13:09:47.603 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.MultiProducerConsumer 2023-09-22T13:10:07.647 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.MultiProducerConsumer (20040 ms) 2023-09-22T13:10:07.647 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.ReserveSpillover 2023-09-22T13:10:11.672 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.ReserveSpillover (4029 ms) 2023-09-22T13:10:11.672 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.CommitSpillover 2023-09-22T13:10:16.695 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.CommitSpillover (5023 ms) 2023-09-22T13:10:16.695 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestCls2PCQueue.AbortSpillover 2023-09-22T13:10:21.717 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestCls2PCQueue.AbortSpillover (5022 ms) 2023-09-22T13:10:21.717 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 22 tests from TestCls2PCQueue (99947 ms total) 2023-09-22T13:10:21.717 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:21.717 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:21.717 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 22 tests from 1 test suite ran. (99947 ms total) 2023-09-22T13:10:21.717 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 22 tests. 2023-09-22T13:10:21.720 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:10:21.720 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:10:21.789 INFO:tasks.workunit:Running workunits matching rgw/test_rgw_gc_log.sh on client.0... 2023-09-22T13:10:21.791 INFO:tasks.workunit:Running workunit rgw/test_rgw_gc_log.sh... 2023-09-22T13:10:21.791 DEBUG:teuthology.orchestra.run.smithi170:workunit test rgw/test_rgw_gc_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_gc_log.sh 2023-09-22T13:10:21.871 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:21.871 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:21.871 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 6 tests from 1 test suite. 2023-09-22T13:10:21.871 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:23.737 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 6 tests from rgw_gc_log 2023-09-22T13:10:23.737 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] rgw_gc_log.init_existing_queue 2023-09-22T13:10:23.746 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] rgw_gc_log.init_existing_queue (10 ms) 2023-09-22T13:10:23.747 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] rgw_gc_log.init_existing_omap 2023-09-22T13:10:23.751 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] rgw_gc_log.init_existing_omap (4 ms) 2023-09-22T13:10:23.751 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] rgw_gc_log.enqueue1_after_init 2023-09-22T13:10:23.754 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] rgw_gc_log.enqueue1_after_init (4 ms) 2023-09-22T13:10:23.754 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] rgw_gc_log.enqueue2_before_init 2023-09-22T13:10:23.756 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] rgw_gc_log.enqueue2_before_init (1 ms) 2023-09-22T13:10:23.756 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] rgw_gc_log.defer1_after_init 2023-09-22T13:10:23.759 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] rgw_gc_log.defer1_after_init (3 ms) 2023-09-22T13:10:23.759 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] rgw_gc_log.defer2_before_init 2023-09-22T13:10:23.760 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] rgw_gc_log.defer2_before_init (2 ms) 2023-09-22T13:10:23.760 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 6 tests from rgw_gc_log (24 ms total) 2023-09-22T13:10:23.760 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:23.760 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:24.740 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 6 tests from 1 test suite ran. (2868 ms total) 2023-09-22T13:10:24.740 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 6 tests. 2023-09-22T13:10:24.743 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:10:24.743 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:10:24.817 INFO:tasks.workunit:Running workunits matching rgw/test_rgw_obj.sh on client.0... 2023-09-22T13:10:24.818 INFO:tasks.workunit:Running workunit rgw/test_rgw_obj.sh... 2023-09-22T13:10:24.819 DEBUG:teuthology.orchestra.run.smithi170:workunit test rgw/test_rgw_obj.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_obj.sh 2023-09-22T13:10:24.902 INFO:tasks.workunit.client.0.smithi170.stdout:Running main() from gmock_main.cc 2023-09-22T13:10:24.903 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 4 tests from 1 test suite. 2023-09-22T13:10:24.903 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:24.903 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 4 tests from TestRGWObj 2023-09-22T13:10:24.903 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestRGWObj.underscore 2023-09-22T13:10:24.903 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.903 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.904 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns= i= 2023-09-22T13:10:24.904 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.904 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.904 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns= i= 2023-09-22T13:10:24.904 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.904 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.904 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns= i= 2023-09-22T13:10:24.904 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.904 INFO:tasks.workunit.client.0.smithi170.stdout: "obj1": { 2023-09-22T13:10:24.905 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.905 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.905 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.905 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.905 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.905 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.905 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.905 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.905 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.906 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.906 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.906 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.906 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.906 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.906 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.906 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.906 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.907 INFO:tasks.workunit.client.0.smithi170.stdout: "obj2": { 2023-09-22T13:10:24.907 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.907 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.907 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.907 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.907 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.907 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.907 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.907 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.908 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.908 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.908 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.908 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.908 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.908 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.908 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.908 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.908 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.909 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3": { 2023-09-22T13:10:24.909 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.909 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.909 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.909 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.909 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.909 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.909 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.909 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.910 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.910 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.910 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.910 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.910 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.910 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.910 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.910 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.910 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.911 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3-2": { 2023-09-22T13:10:24.911 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.911 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.911 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.911 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.911 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.911 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.911 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.911 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.912 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.912 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.912 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.912 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.912 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.912 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.912 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.912 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.912 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.913 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.913 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.913 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.913 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns=ns i= 2023-09-22T13:10:24.913 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.913 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.913 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns=ns i= 2023-09-22T13:10:24.913 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.913 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj 2023-09-22T13:10:24.914 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns=ns i= 2023-09-22T13:10:24.914 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.914 INFO:tasks.workunit.client.0.smithi170.stdout: "obj1": { 2023-09-22T13:10:24.914 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.914 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.914 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.914 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.914 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.915 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.915 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.915 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.915 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.915 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.915 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.915 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.915 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.915 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.916 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.916 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.916 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.916 INFO:tasks.workunit.client.0.smithi170.stdout: "obj2": { 2023-09-22T13:10:24.916 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.916 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.916 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.916 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.916 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.916 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.917 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.917 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.917 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.917 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.917 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.917 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.917 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.917 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.917 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.918 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.918 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.918 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3": { 2023-09-22T13:10:24.918 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.918 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.918 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.918 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.918 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.918 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.919 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.919 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.919 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.919 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.919 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.919 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.919 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.919 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.919 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.920 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.920 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.920 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3-2": { 2023-09-22T13:10:24.920 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.920 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.920 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.920 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.920 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.920 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.920 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.921 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.921 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.921 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.921 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.921 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.921 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.921 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.921 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.921 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.922 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.922 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.922 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.922 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.922 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns= i=v1 2023-09-22T13:10:24.922 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.922 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.922 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns= i=v1 2023-09-22T13:10:24.922 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.923 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.923 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns= i=v1 2023-09-22T13:10:24.923 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.923 INFO:tasks.workunit.client.0.smithi170.stdout: "obj1": { 2023-09-22T13:10:24.923 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.923 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.923 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.923 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.923 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.924 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.924 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.924 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.924 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.924 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.924 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.924 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.924 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.924 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.925 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.925 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.925 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.925 INFO:tasks.workunit.client.0.smithi170.stdout: "obj2": { 2023-09-22T13:10:24.925 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.925 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.925 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.925 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.925 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.925 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.926 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.926 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.926 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.926 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.926 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.926 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.926 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.926 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.926 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.927 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.927 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.927 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3": { 2023-09-22T13:10:24.927 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.927 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.927 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.927 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.927 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.927 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.927 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.928 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.928 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.928 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.928 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.928 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.928 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.928 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.928 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.928 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.929 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.929 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3-2": { 2023-09-22T13:10:24.929 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.929 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.929 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.929 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.929 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.929 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.929 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.929 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.929 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.930 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.930 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.930 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.930 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.930 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.930 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.930 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.930 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.930 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.931 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.931 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.931 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns=ns i=v1 2023-09-22T13:10:24.931 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.931 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.931 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns=ns i=v1 2023-09-22T13:10:24.931 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.931 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: _obj[v1] 2023-09-22T13:10:24.931 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: _obj ns=ns i=v1 2023-09-22T13:10:24.931 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.932 INFO:tasks.workunit.client.0.smithi170.stdout: "obj1": { 2023-09-22T13:10:24.932 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.932 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.932 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.932 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.932 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.932 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.932 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.932 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.932 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.933 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.933 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.933 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.933 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.933 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.933 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.933 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.933 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.933 INFO:tasks.workunit.client.0.smithi170.stdout: "obj2": { 2023-09-22T13:10:24.933 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.934 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.934 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.934 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.934 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.934 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.934 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.934 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.934 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.934 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.935 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.935 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.935 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.935 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.935 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.935 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.935 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.935 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3": { 2023-09-22T13:10:24.935 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.935 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.936 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.936 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.936 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.936 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.936 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.936 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.936 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.936 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.936 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.936 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.937 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.937 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.937 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.937 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.937 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.937 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3-2": { 2023-09-22T13:10:24.937 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.937 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.937 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.937 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.937 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.938 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.938 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.938 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.938 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.938 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.938 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.938 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.938 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_obj", 2023-09-22T13:10:24.938 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.939 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.939 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.939 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.939 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.939 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestRGWObj.underscore (0 ms) 2023-09-22T13:10:24.939 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestRGWObj.no_underscore 2023-09-22T13:10:24.939 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.939 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.939 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns= i= 2023-09-22T13:10:24.939 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.940 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.940 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns= i= 2023-09-22T13:10:24.940 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.940 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.940 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns= i= 2023-09-22T13:10:24.940 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.940 INFO:tasks.workunit.client.0.smithi170.stdout: "obj1": { 2023-09-22T13:10:24.940 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.940 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.940 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.940 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.941 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.941 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.941 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.941 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.941 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.941 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.941 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.941 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.941 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.941 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.941 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.942 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.942 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.942 INFO:tasks.workunit.client.0.smithi170.stdout: "obj2": { 2023-09-22T13:10:24.942 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.942 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.942 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.942 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.942 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.942 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.943 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.943 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.943 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.943 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.943 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.943 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.943 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.943 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.943 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.943 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.943 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.944 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3": { 2023-09-22T13:10:24.944 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.944 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.944 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.944 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.944 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.944 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.944 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.944 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.945 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.945 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.945 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.945 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.945 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.945 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.945 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.945 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.945 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.945 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3-2": { 2023-09-22T13:10:24.946 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.946 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.946 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.946 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.946 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.946 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.946 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.946 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.946 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.946 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.947 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.947 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.947 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.947 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.947 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.947 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.947 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.947 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.947 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.947 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.948 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns=ns i= 2023-09-22T13:10:24.948 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.948 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.948 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns=ns i= 2023-09-22T13:10:24.948 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.948 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj 2023-09-22T13:10:24.948 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns=ns i= 2023-09-22T13:10:24.948 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.948 INFO:tasks.workunit.client.0.smithi170.stdout: "obj1": { 2023-09-22T13:10:24.948 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.949 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.949 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.949 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.949 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.949 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.949 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.949 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.949 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.949 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.949 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.949 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.950 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.950 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.950 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.950 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.950 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.950 INFO:tasks.workunit.client.0.smithi170.stdout: "obj2": { 2023-09-22T13:10:24.950 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.950 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.950 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.950 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.951 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.951 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.951 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.951 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.951 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.951 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.951 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.951 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.951 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.951 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.951 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.952 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.952 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.952 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3": { 2023-09-22T13:10:24.952 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.952 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.952 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.952 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.952 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.952 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.952 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.952 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.953 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.953 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.953 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.953 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.953 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.953 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.953 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.953 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.953 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.953 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3-2": { 2023-09-22T13:10:24.954 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.954 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.954 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.954 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.954 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.954 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.954 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.954 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.954 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.954 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.955 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.955 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.955 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.955 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.955 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.955 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.955 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.955 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.955 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.955 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.955 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns= i=v1 2023-09-22T13:10:24.956 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.956 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.956 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns= i=v1 2023-09-22T13:10:24.956 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.956 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.956 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns= i=v1 2023-09-22T13:10:24.956 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.956 INFO:tasks.workunit.client.0.smithi170.stdout: "obj1": { 2023-09-22T13:10:24.956 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.956 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.956 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.957 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.957 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.957 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.957 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.957 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.957 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.957 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.957 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.957 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.957 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.957 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.958 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.958 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.958 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.958 INFO:tasks.workunit.client.0.smithi170.stdout: "obj2": { 2023-09-22T13:10:24.958 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.958 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.958 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.958 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.958 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.958 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.959 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.959 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.959 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.959 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.959 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.959 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.959 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.959 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.959 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.959 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.959 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.960 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3": { 2023-09-22T13:10:24.960 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.960 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.960 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.960 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.960 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.960 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.960 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.960 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.960 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.960 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3-2": { 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.961 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.962 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.962 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.962 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.962 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.962 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.962 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.962 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.962 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.962 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.962 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.963 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.963 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.963 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.963 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.963 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.963 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.963 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns=ns i=v1 2023-09-22T13:10:24.963 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.963 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.963 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns=ns i=v1 2023-09-22T13:10:24.963 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.964 INFO:tasks.workunit.client.0.smithi170.stdout:parsed: obj[v1] 2023-09-22T13:10:24.964 INFO:tasks.workunit.client.0.smithi170.stdout:stripped: obj ns=ns i=v1 2023-09-22T13:10:24.964 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.964 INFO:tasks.workunit.client.0.smithi170.stdout: "obj1": { 2023-09-22T13:10:24.964 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.964 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.964 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.964 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.964 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.964 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.964 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.965 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.965 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.965 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.965 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.965 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.965 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.965 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.965 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.965 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.965 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.965 INFO:tasks.workunit.client.0.smithi170.stdout: "obj2": { 2023-09-22T13:10:24.966 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.966 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.966 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.966 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.966 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.966 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.966 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.966 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.966 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.966 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.966 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.967 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.967 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.967 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.967 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.967 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.967 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.967 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3": { 2023-09-22T13:10:24.967 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.967 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.967 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.967 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.968 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.968 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.968 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.968 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.968 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.968 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.968 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.968 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.968 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.968 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.968 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: "obj3-2": { 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.969 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.970 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.970 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.970 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.970 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "obj", 2023-09-22T13:10:24.970 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "v1", 2023-09-22T13:10:24.970 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.970 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.970 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.970 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.970 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestRGWObj.no_underscore (0 ms) 2023-09-22T13:10:24.970 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestRGWObj.obj_to_raw 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.971 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.972 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.972 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.972 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:24.972 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.972 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.972 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.972 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.972 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.972 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.972 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.972 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.973 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:24.974 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:24.974 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker_myobj", 2023-09-22T13:10:24.974 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:24.974 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.974 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.974 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.974 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:24.974 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.974 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.974 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.975 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.975 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.975 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.975 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.975 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.975 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.975 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.975 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.975 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.975 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:24.975 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.976 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.976 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.976 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.976 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.976 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.976 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.976 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:24.976 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.976 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.976 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.976 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.977 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.978 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.978 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.978 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:24.978 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.978 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:24.978 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.978 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.978 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.978 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:24.978 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.978 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.979 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.979 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.979 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.979 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.979 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:24.979 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:24.979 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker_myobj", 2023-09-22T13:10:24.979 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:24.979 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.979 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.979 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.980 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.981 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.982 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.982 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.982 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.982 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.982 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.982 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.982 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.982 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.982 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.982 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:24.982 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.983 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.983 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.983 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.983 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.983 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.983 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.983 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.983 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.983 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.983 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.983 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:24.984 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.985 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.985 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.985 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:24.985 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.985 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.985 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.985 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.985 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.985 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.985 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.985 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.986 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.986 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.986 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.986 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.986 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:24.986 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.986 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.986 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.986 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.986 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.986 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.987 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.987 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:24.987 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.987 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.987 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.987 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.987 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:24.987 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.987 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:24.987 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.987 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.988 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.988 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.988 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:24.988 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.988 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:24.988 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.988 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.988 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.988 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.988 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:24.988 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.989 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.990 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.991 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.991 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:24.991 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.991 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:24.991 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.991 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.991 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.991 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:24.991 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:24.991 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:24.991 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.992 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.992 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.992 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.992 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.992 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.992 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.992 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.992 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.992 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.992 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.992 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.993 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.994 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.994 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.994 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.994 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.994 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.994 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.994 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:24.994 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:24.994 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.994 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:24.995 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.996 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.997 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.998 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:24.998 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.998 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:24.998 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.998 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.998 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:24.998 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:24.998 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:24.998 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:24.998 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:24.999 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.000 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.001 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.002 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.003 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.004 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.005 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.006 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.006 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.006 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.006 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.006 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.006 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.006 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.006 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.006 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.006 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.006 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.007 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.008 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.008 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.008 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.008 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.008 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.008 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.008 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.008 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.008 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.008 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.008 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.009 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.010 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.011 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.012 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.013 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.014 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.015 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.015 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.015 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.015 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.015 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.015 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.015 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.015 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.015 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:25.015 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker___myobj", 2023-09-22T13:10:25.015 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj" 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.016 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.017 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.018 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.019 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker___myobj", 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj" 2023-09-22T13:10:25.020 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.021 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.022 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.023 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.024 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.025 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.026 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.027 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.028 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.028 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.028 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.028 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.028 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.028 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.028 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.028 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.028 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.028 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.028 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.029 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:10:25.030 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.031 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.032 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.033 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.034 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj" 2023-09-22T13:10:25.035 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.036 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.037 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.038 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.038 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.038 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.038 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.038 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.038 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.038 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.038 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.038 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.038 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.038 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.039 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj" 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.040 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.041 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.042 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.043 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.043 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.043 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.043 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.043 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.043 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.043 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.043 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.043 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.043 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.043 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.044 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.045 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.046 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.047 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.048 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.049 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.050 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.051 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.052 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.053 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:25.054 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker___myobj_", 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj_" 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.055 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.056 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.056 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.056 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.056 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.056 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.056 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.056 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.056 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.056 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.056 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.057 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.057 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.057 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.057 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.057 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.057 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.057 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.057 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.057 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.058 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.058 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.058 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.058 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.058 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.058 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.058 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.058 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.058 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.059 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.059 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.059 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.059 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.059 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.059 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.059 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.059 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.059 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.059 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.060 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.060 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.060 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.060 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.060 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.060 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.060 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.060 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.060 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.060 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker___myobj_", 2023-09-22T13:10:25.061 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj_" 2023-09-22T13:10:25.061 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.061 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.061 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.061 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.061 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.061 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.061 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.061 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.061 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.061 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.062 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.062 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.062 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.062 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.062 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.062 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.062 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.062 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.062 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.062 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.063 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.063 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.063 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.063 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.063 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.063 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.063 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.063 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.063 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.063 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.064 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.064 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.064 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.064 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.064 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.064 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.064 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.064 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.064 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.064 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.065 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.065 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.065 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.065 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.065 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.065 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.065 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.065 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.065 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.065 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.065 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.066 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.066 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.066 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.066 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.066 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.066 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.066 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:25.066 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:10:25.066 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.066 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.067 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.067 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.067 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.067 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.067 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.067 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.067 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.067 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.067 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.067 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.067 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.068 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.068 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.068 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.068 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.068 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.068 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.068 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.068 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.068 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.068 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.068 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.069 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.069 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.069 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.069 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.069 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.069 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.069 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.069 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.069 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.069 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.069 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.070 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.070 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.070 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.070 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.070 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.070 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.070 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.070 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.070 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.070 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.071 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.071 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.071 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.071 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.071 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.071 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.071 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.071 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.071 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.071 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.071 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.072 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.072 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.072 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.072 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:10:25.072 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.072 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.072 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.072 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.072 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.072 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.072 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.073 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.073 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.073 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.073 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.073 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.073 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.073 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.073 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.073 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.073 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.073 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.074 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.074 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.074 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.074 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.074 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.074 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.074 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.074 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.074 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.074 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.075 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.075 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.075 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.075 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.075 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.075 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.075 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.075 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.075 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.075 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.076 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.076 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.076 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.076 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.076 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.076 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.076 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.076 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.076 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.076 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.076 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.077 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.077 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.077 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.077 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.077 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.077 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.077 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.077 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.077 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.077 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:25.077 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:10:25.078 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj_" 2023-09-22T13:10:25.078 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.078 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.078 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.078 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.078 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.078 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.078 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.078 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.078 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.079 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.079 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.079 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.079 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.079 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.079 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.079 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.079 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.079 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.079 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.080 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.080 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.080 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.080 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.080 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.080 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.080 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.080 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.080 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.080 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.080 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.081 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.082 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.082 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.082 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.082 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.082 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.082 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.082 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.082 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.082 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.083 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.083 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.083 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.083 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.083 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.083 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.083 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.083 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:10:25.083 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj_" 2023-09-22T13:10:25.083 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.083 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.084 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.084 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.084 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.084 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.084 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.084 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.084 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.084 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.084 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.084 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.085 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.085 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.085 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.085 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.085 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.085 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.085 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.085 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.085 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.085 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.085 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.086 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.086 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.086 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.086 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.086 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.086 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.086 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.086 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.086 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.086 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.087 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.087 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.087 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.087 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.087 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.087 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.087 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.087 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.087 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.087 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.087 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.088 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.088 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.088 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.088 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.088 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.088 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.088 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.088 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.088 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.088 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.088 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": "default-placement.data", 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "test", 2023-09-22T13:10:25.089 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.090 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.090 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.090 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.090 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": "", 2023-09-22T13:10:25.090 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.090 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": "" 2023-09-22T13:10:25.090 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.090 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.090 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.090 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.091 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.091 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.091 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.091 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.091 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.091 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.091 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.091 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.091 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.091 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.092 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.092 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.092 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.092 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.092 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.092 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.092 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.092 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.092 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.092 INFO:tasks.workunit.client.0.smithi170.stdout: "obj": { 2023-09-22T13:10:25.092 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.093 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.093 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.093 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.093 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.093 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.093 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.093 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.093 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.093 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.093 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.093 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.094 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.094 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.094 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.094 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.094 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.094 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.094 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.094 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.094 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.094 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:10:25.095 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.095 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.095 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.095 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.095 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.095 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.095 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.095 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.095 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.095 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.096 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.096 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.096 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.096 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.096 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.096 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.096 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.096 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.096 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.096 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.096 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.097 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.097 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.097 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestRGWObj.obj_to_raw (1 ms) 2023-09-22T13:10:25.097 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] TestRGWObj.old_to_raw 2023-09-22T13:10:25.097 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.097 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.097 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.097 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker_myobj", 2023-09-22T13:10:25.097 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.097 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.097 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.098 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.098 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.098 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.098 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.098 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.098 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.098 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.098 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.098 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.098 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.098 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.099 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.099 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.099 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.099 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.099 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.099 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.099 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.099 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.099 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.099 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker_myobj 2023-09-22T13:10:25.099 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.100 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.100 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.100 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker_myobj", 2023-09-22T13:10:25.100 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.100 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.100 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.100 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.100 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.100 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.100 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.100 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.101 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.102 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.102 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.102 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.102 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker_myobj 2023-09-22T13:10:25.102 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.102 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.102 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.102 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:10:25.102 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.102 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.102 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.103 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.103 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.103 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.103 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.103 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.103 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.103 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.103 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.103 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.103 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.103 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.104 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.104 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.104 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.104 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.104 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.104 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.104 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.104 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.104 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.104 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker__ns_myobj 2023-09-22T13:10:25.104 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.105 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.105 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.105 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:10:25.105 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.105 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.105 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.105 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.105 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.105 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.105 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.105 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.106 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.106 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.106 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.106 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.106 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.106 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.106 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.106 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.106 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.106 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.106 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker__ns_myobj 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.107 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.108 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.108 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.108 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.108 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.108 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.108 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.108 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.108 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.108 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.108 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.108 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.109 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.109 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.109 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.109 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.109 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.109 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.109 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.109 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.109 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.109 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker__:inst_myobj 2023-09-22T13:10:25.109 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.110 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.110 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.110 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:10:25.110 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.110 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.110 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.110 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.110 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.110 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.110 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.110 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.111 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.111 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.111 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.111 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.111 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.111 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.111 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.111 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.111 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.111 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.111 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker__:inst_myobj 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.112 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.113 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.113 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.113 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.113 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.113 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.113 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.113 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.113 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.113 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.113 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.113 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.114 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.114 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.114 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.114 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.114 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.114 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.114 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.114 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.114 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.114 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker__ns:inst_myobj 2023-09-22T13:10:25.114 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.115 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.115 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.115 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:10:25.115 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.115 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.115 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.115 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.115 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.115 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.115 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.116 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.116 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.116 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.116 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.116 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.116 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.116 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.116 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.116 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.116 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.116 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "myobj", 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker__ns:inst_myobj 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker___myobj", 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj" 2023-09-22T13:10:25.117 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.118 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.118 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.118 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.118 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.118 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.118 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.118 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.118 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.118 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.118 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.118 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker___myobj 2023-09-22T13:10:25.119 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.120 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.120 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.120 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker___myobj", 2023-09-22T13:10:25.120 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj" 2023-09-22T13:10:25.120 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.120 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.120 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.120 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.120 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.120 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.120 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.121 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.122 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.122 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.122 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.122 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker___myobj 2023-09-22T13:10:25.122 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.122 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.122 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.122 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:10:25.122 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.122 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.122 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.123 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.123 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.123 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.123 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.123 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.123 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.123 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.123 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.123 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.123 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.123 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker__ns__myobj 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.124 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.125 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.126 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.126 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.126 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.126 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.126 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.126 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.126 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.126 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.126 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.126 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.126 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.127 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.127 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.127 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker__ns__myobj 2023-09-22T13:10:25.127 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.127 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.127 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.127 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:10:25.127 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj" 2023-09-22T13:10:25.127 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.128 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.129 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.129 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.129 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.129 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.129 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.129 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.129 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.129 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.129 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.129 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.129 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.130 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.130 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.130 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.130 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.130 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.130 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.130 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.130 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.130 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.130 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker__:inst__myobj 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj" 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.131 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.132 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.132 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.132 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.132 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.132 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.132 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.132 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.132 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.132 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.132 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.132 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.133 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.133 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.133 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.133 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.133 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker__:inst__myobj 2023-09-22T13:10:25.133 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.133 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.133 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.133 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:10:25.133 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.133 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.134 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.134 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.134 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.134 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.134 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.134 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.134 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.134 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.134 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.134 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.134 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.135 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.135 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.135 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.135 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.135 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.135 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.135 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.135 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.135 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.135 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.135 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker__ns:inst__myobj 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.136 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.137 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.137 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.137 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.137 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.137 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.137 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.137 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.137 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.137 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.137 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj", 2023-09-22T13:10:25.137 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.138 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.138 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.138 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.138 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.138 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker__ns:inst__myobj 2023-09-22T13:10:25.138 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.138 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.138 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.138 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker___myobj_", 2023-09-22T13:10:25.138 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj_" 2023-09-22T13:10:25.138 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.139 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.139 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.139 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.139 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.139 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.139 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.139 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.139 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.139 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.139 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.139 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker___myobj_ 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.140 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker___myobj_", 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj_" 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.141 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.142 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker___myobj_ 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.143 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.144 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker__ns__myobj_ 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.145 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "", 2023-09-22T13:10:25.146 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker__ns__myobj_ 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj_" 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.147 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.148 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker__:inst__myobj_ 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "marker__myobj_" 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.149 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.150 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "" 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker__:inst__myobj_ 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj": { 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.151 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj": { 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.152 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout:raw=.data-pool:marker__ns:inst__myobj_ 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout: "raw_obj2": { 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout: "pool": ".data-pool", 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:10:25.153 INFO:tasks.workunit.client.0.smithi170.stdout: "loc": "" 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout: "new_obj2": { 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket": { 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "ebtest", 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout: "marker": "marker", 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_id": "bucket-id", 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout: "tenant": "", 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout: "explicit_placement": { 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout: "data_pool": ".data-pool", 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout: "data_extra_pool": "", 2023-09-22T13:10:25.154 INFO:tasks.workunit.client.0.smithi170.stdout: "index_pool": ".index-pool" 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout: "key": { 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout: "name": "_myobj_", 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout: "instance": "inst", 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout: "ns": "ns" 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout:raw2=.data-pool:marker__ns:inst__myobj_ 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] TestRGWObj.old_to_raw (0 ms) 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 4 tests from TestRGWObj (1 ms total) 2023-09-22T13:10:25.155 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:25.156 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:25.156 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 4 tests from 1 test suite ran. (1 ms total) 2023-09-22T13:10:25.156 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 4 tests. 2023-09-22T13:10:25.156 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:10:25.156 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:10:25.189 INFO:tasks.workunit:Running workunits matching rgw/test_rgw_throttle.sh on client.0... 2023-09-22T13:10:25.190 INFO:tasks.workunit:Running workunit rgw/test_rgw_throttle.sh... 2023-09-22T13:10:25.191 DEBUG:teuthology.orchestra.run.smithi170:workunit test rgw/test_rgw_throttle.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_throttle.sh 2023-09-22T13:10:25.277 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:25.277 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:25.277 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 5 tests from 1 test suite. 2023-09-22T13:10:25.277 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:26.758 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 5 tests from Aio_Throttle 2023-09-22T13:10:26.758 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] Aio_Throttle.NoThrottleUpToMax 2023-09-22T13:10:26.758 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] Aio_Throttle.NoThrottleUpToMax (0 ms) 2023-09-22T13:10:26.758 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] Aio_Throttle.CostOverWindow 2023-09-22T13:10:26.758 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] Aio_Throttle.CostOverWindow (0 ms) 2023-09-22T13:10:26.759 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] Aio_Throttle.ThrottleOverMax 2023-09-22T13:10:26.845 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] Aio_Throttle.ThrottleOverMax (88 ms) 2023-09-22T13:10:26.846 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] Aio_Throttle.YieldCostOverWindow 2023-09-22T13:10:26.846 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] Aio_Throttle.YieldCostOverWindow (0 ms) 2023-09-22T13:10:26.846 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] Aio_Throttle.YieldingThrottleOverMax 2023-09-22T13:10:26.933 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] Aio_Throttle.YieldingThrottleOverMax (88 ms) 2023-09-22T13:10:26.933 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 5 tests from Aio_Throttle (176 ms total) 2023-09-22T13:10:26.934 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:26.934 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:26.934 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 5 tests from 1 test suite ran. (1657 ms total) 2023-09-22T13:10:26.935 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 5 tests. 2023-09-22T13:10:26.938 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:10:26.938 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:10:26.969 INFO:tasks.workunit:Running workunits matching rgw/test_librgw_file.sh on client.0... 2023-09-22T13:10:26.970 INFO:tasks.workunit:Running workunit rgw/test_librgw_file.sh... 2023-09-22T13:10:26.970 DEBUG:teuthology.orchestra.run.smithi170:workunit test rgw/test_librgw_file.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_librgw_file.sh 2023-09-22T13:10:27.069 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:27.070 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:27.077 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.078+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.079 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.080+0000 7f4dfc132500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:27.080 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.080+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.080 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.081+0000 7f4dfc132500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:27.080 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.081+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.081 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.081+0000 7f4dfc132500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:27.081 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.081+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.082 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.082+0000 7f4dfc132500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:27.093 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.093+0000 7f4dfc132500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:10:27.094 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.093+0000 7f4dfc132500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:10:27.094 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.093+0000 7f4dfc132500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:10:27.094 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.093+0000 7f4dfc132500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:10:27.094 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.093+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.094 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.093+0000 7f4dfc132500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:27.094 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.093+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.094 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.094+0000 7f4dfc132500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:10:27.095 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.094+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.095 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.094+0000 7f4dfc132500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:27.095 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.094+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.097 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.094+0000 7f4dfc132500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:10:27.097 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.094+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.098 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.095+0000 7f4dfc132500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:27.098 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.095+0000 7f4dfc132500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:10:27.098 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.095+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.098 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.095+0000 7f4dfc132500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:27.098 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.095+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.098 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.095+0000 7f4dfc132500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:27.098 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.095+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.098 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.095+0000 7f4dfc132500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:10:27.099 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.095+0000 7f4dfc132500 20 zonegroup default 2023-09-22T13:10:27.099 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.095+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.099 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.096+0000 7f4dfc132500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:27.099 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.096+0000 7f4dfc132500 10 Cannot find current period zone using local zone 2023-09-22T13:10:27.099 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.096+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.099 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.096+0000 7f4dfc132500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:27.099 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.096+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.099 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.096+0000 7f4dfc132500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:27.099 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.096+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.100 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.096+0000 7f4dfc132500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:10:27.100 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.096+0000 7f4dfc132500 20 zone default found 2023-09-22T13:10:27.100 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.096+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.100 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.097+0000 7f4dfc132500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:27.100 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.097+0000 7f4dfc132500 20 rados->read ofs=0 len=0 2023-09-22T13:10:27.100 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.097+0000 7f4dfc132500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:27.100 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.097+0000 7f4dfc132500 20 started sync module instance, tier type = 2023-09-22T13:10:27.100 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.097+0000 7f4dfc132500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:10:27.107 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.107+0000 7f4dfc132500 20 add_watcher() i=0 2023-09-22T13:10:27.107 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.107+0000 7f4dfc132500 20 add_watcher() i=1 2023-09-22T13:10:27.107 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.107+0000 7f4dfc132500 20 add_watcher() i=2 2023-09-22T13:10:27.107 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.107+0000 7f4dfc132500 20 add_watcher() i=3 2023-09-22T13:10:27.107 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.107+0000 7f4dfc132500 20 add_watcher() i=4 2023-09-22T13:10:27.108 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.107+0000 7f4dfc132500 20 add_watcher() i=5 2023-09-22T13:10:27.108 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.107+0000 7f4dfc132500 20 add_watcher() i=6 2023-09-22T13:10:27.109 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.110+0000 7f4dfc132500 20 add_watcher() i=7 2023-09-22T13:10:27.109 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.110+0000 7f4dfc132500 2 all 8 watchers are set, enabling cache 2023-09-22T13:10:27.110 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.110+0000 7f4de7e15700 0 -- 172.21.15.170:0/1641772009 >> [v2:172.21.15.191:6816/144992,v1:172.21.15.191:6817/144992] conn(0x7f4ddc026040 msgr2=0x7f4ddc025410 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:10:27.112 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.112+0000 7f4d33fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:10:27.112 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.112+0000 7f4dfc132500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:10:27.112 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.112+0000 7f4dfc132500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:10:27.116 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.116+0000 7f4de7614700 0 -- 172.21.15.170:0/1224058527 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x5587c4911eb0 msgr2=0x5587c4909060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:10:27.118 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.118+0000 7f4dfc132500 5 note: GC not initialized 2023-09-22T13:10:27.120 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.119+0000 7f4d2affd700 20 reqs_thread_entry: start 2023-09-22T13:10:27.120 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.119+0000 7f4d28ff9700 20 reqs_thread_entry: start 2023-09-22T13:10:27.120 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.119+0000 7f4dfc132500 20 init_complete bucket index max shards: 11 2023-09-22T13:10:27.120 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.120+0000 7f4dfc132500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:10:27.120 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.120+0000 7f4d16ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:57 2023 2023-09-22T13:10:27.120 INFO:tasks.workunit.client.0.smithi170.stderr: 2023-09-22T13:10:27.121 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.120+0000 7f4d167fc700 20 reqs_thread_entry: start 2023-09-22T13:10:27.121 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.120+0000 7f4dfc132500 20 get_system_obj_state: rctx=0x7fffaf3f9a78 obj=default.rgw.meta:users.uid:ceph-test-librgw-file state=0x5587c4880cd0 s->prefetch_data=0 2023-09-22T13:10:27.121 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.120+0000 7f4dfc132500 10 cache get: name=default.rgw.meta+users.uid+ceph-test-librgw-file : miss 2023-09-22T13:10:27.121 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.120+0000 7f4dfc132500 10 cache put: name=default.rgw.meta+users.uid+ceph-test-librgw-file info.flags=0x0 2023-09-22T13:10:27.121 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.120+0000 7f4dfc132500 10 adding default.rgw.meta+users.uid+ceph-test-librgw-file to cache LRU end 2023-09-22T13:10:27.121 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.120+0000 7f4dfc132500 20 get_system_obj_state: rctx=0x7fffaf3f9a88 obj=default.rgw.meta:users.email:librgw@example.com state=0x5587c4880cd0 s->prefetch_data=0 2023-09-22T13:10:27.121 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.120+0000 7f4dfc132500 10 cache get: name=default.rgw.meta+users.email+librgw@example.com : miss 2023-09-22T13:10:27.123 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.121+0000 7f4dfc132500 10 cache put: name=default.rgw.meta+users.email+librgw@example.com info.flags=0x0 2023-09-22T13:10:27.123 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.121+0000 7f4dfc132500 10 adding default.rgw.meta+users.email+librgw@example.com to cache LRU end 2023-09-22T13:10:27.123 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.121+0000 7f4dfc132500 20 get_system_obj_state: rctx=0x7fffaf3f9a88 obj=default.rgw.meta:users.keys:sTh5FWA08eT6gR4jiOiw+p5XB6g= state=0x5587c4880cd0 s->prefetch_data=0 2023-09-22T13:10:27.123 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.121+0000 7f4dfc132500 10 cache get: name=default.rgw.meta+users.keys+sTh5FWA08eT6gR4jiOiw+p5XB6g= : miss 2023-09-22T13:10:27.123 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.121+0000 7f4dfc132500 10 cache put: name=default.rgw.meta+users.keys+sTh5FWA08eT6gR4jiOiw+p5XB6g= info.flags=0x0 2023-09-22T13:10:27.123 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.121+0000 7f4dfc132500 10 adding default.rgw.meta+users.keys+sTh5FWA08eT6gR4jiOiw+p5XB6g= to cache LRU end 2023-09-22T13:10:27.123 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.121+0000 7f4dfc132500 20 get_system_obj_state: rctx=0x7fffaf3f9388 obj=default.rgw.meta:users.keys:sTh5FWA08eT6gR4jiOiw+p5XB6g= state=0x5587c4880cd0 s->prefetch_data=0 2023-09-22T13:10:27.124 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.121+0000 7f4dfc132500 10 cache get: name=default.rgw.meta+users.keys+sTh5FWA08eT6gR4jiOiw+p5XB6g= : hit (negative entry) 2023-09-22T13:10:27.124 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.121+0000 7f4dfc132500 20 get_system_obj_state: rctx=0x7fffaf3f96c8 obj=default.rgw.meta:users.keys:sTh5FWA08eT6gR4jiOiw+p5XB6g= state=0x5587c4880cd0 s->prefetch_data=0 2023-09-22T13:10:27.124 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.121+0000 7f4dfc132500 10 cache get: name=default.rgw.meta+users.keys+sTh5FWA08eT6gR4jiOiw+p5XB6g= : hit (negative entry) 2023-09-22T13:10:27.124 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.122+0000 7f4dfc132500 10 cache put: name=default.rgw.meta+users.uid+ceph-test-librgw-file info.flags=0x17 2023-09-22T13:10:27.124 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.122+0000 7f4dfc132500 10 moving default.rgw.meta+users.uid+ceph-test-librgw-file to cache LRU end 2023-09-22T13:10:27.124 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.122+0000 7f4dfc132500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.uid:ceph-test-librgw-file, ofs0, ns] 2023-09-22T13:10:27.124 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.122+0000 7f4de5610700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1013612281858 cookie 94041606556288 notifier 5195 bl.length()=663 2023-09-22T13:10:27.124 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.122+0000 7f4de5610700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+ceph-test-librgw-file info.flags=0x17 2023-09-22T13:10:27.124 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.122+0000 7f4de5610700 10 rgw watcher librados: moving default.rgw.meta+users.uid+ceph-test-librgw-file to cache LRU end 2023-09-22T13:10:27.125 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.124+0000 7f4dfc132500 10 cache put: name=default.rgw.meta+users.email+librgw@example.com info.flags=0x7 2023-09-22T13:10:27.125 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.124+0000 7f4dfc132500 10 moving default.rgw.meta+users.email+librgw@example.com to cache LRU end 2023-09-22T13:10:27.125 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.124+0000 7f4dfc132500 10 distributing notification oid=default.rgw.control:notify.1 cni=[op: 0, obj: default.rgw.meta:users.email:librgw@example.com, ofs0, ns] 2023-09-22T13:10:27.126 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.124+0000 7f4de4e0f700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1013612281856 cookie 94041606560416 notifier 5195 bl.length()=194 2023-09-22T13:10:27.126 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.124+0000 7f4de4e0f700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.email+librgw@example.com info.flags=0x7 2023-09-22T13:10:27.126 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.124+0000 7f4de4e0f700 10 rgw watcher librados: moving default.rgw.meta+users.email+librgw@example.com to cache LRU end 2023-09-22T13:10:27.127 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.126+0000 7f4dfc132500 10 cache put: name=default.rgw.meta+users.keys+sTh5FWA08eT6gR4jiOiw+p5XB6g= info.flags=0x7 2023-09-22T13:10:27.127 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.126+0000 7f4dfc132500 10 moving default.rgw.meta+users.keys+sTh5FWA08eT6gR4jiOiw+p5XB6g= to cache LRU end 2023-09-22T13:10:27.127 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.126+0000 7f4dfc132500 10 distributing notification oid=default.rgw.control:notify.1 cni=[op: 0, obj: default.rgw.meta:users.keys:sTh5FWA08eT6gR4jiOiw+p5XB6g=, ofs0, ns] 2023-09-22T13:10:27.127 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.127+0000 7f4de5610700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1013612281857 cookie 94041606560416 notifier 5195 bl.length()=203 2023-09-22T13:10:27.127 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.127+0000 7f4de5610700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+sTh5FWA08eT6gR4jiOiw+p5XB6g= info.flags=0x7 2023-09-22T13:10:27.127 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.127+0000 7f4de5610700 10 rgw watcher librados: moving default.rgw.meta+users.keys+sTh5FWA08eT6gR4jiOiw+p5XB6g= to cache LRU end 2023-09-22T13:10:27.127 INFO:tasks.workunit.client.0.smithi170.stdout:{ 2023-09-22T13:10:27.128 INFO:tasks.workunit.client.0.smithi170.stdout: "user_id": "ceph-test-librgw-file", 2023-09-22T13:10:27.128 INFO:tasks.workunit.client.0.smithi170.stdout: "display_name": "librgw test user", 2023-09-22T13:10:27.128 INFO:tasks.workunit.client.0.smithi170.stdout: "email": "librgw@example.com", 2023-09-22T13:10:27.128 INFO:tasks.workunit.client.0.smithi170.stdout: "suspended": 0, 2023-09-22T13:10:27.128 INFO:tasks.workunit.client.0.smithi170.stdout: "max_buckets": 1000, 2023-09-22T13:10:27.128 INFO:tasks.workunit.client.0.smithi170.stdout: "subusers": [], 2023-09-22T13:10:27.128 INFO:tasks.workunit.client.0.smithi170.stdout: "keys": [ 2023-09-22T13:10:27.128 INFO:tasks.workunit.client.0.smithi170.stdout: { 2023-09-22T13:10:27.128 INFO:tasks.workunit.client.0.smithi170.stdout: "user": "ceph-test-librgw-file", 2023-09-22T13:10:27.129 INFO:tasks.workunit.client.0.smithi170.stdout: "access_key": "sTh5FWA08eT6gR4jiOiw+p5XB6g=", 2023-09-22T13:10:27.129 INFO:tasks.workunit.client.0.smithi170.stdout: "secret_key": "1fT1fN7Bw+NztawR0s85kHTXx4MHV0t8Nh/+J658OFMokvLgqTr+YA==" 2023-09-22T13:10:27.129 INFO:tasks.workunit.client.0.smithi170.stdout: } 2023-09-22T13:10:27.129 INFO:tasks.workunit.client.0.smithi170.stdout: ], 2023-09-22T13:10:27.129 INFO:tasks.workunit.client.0.smithi170.stdout: "swift_keys": [], 2023-09-22T13:10:27.129 INFO:tasks.workunit.client.0.smithi170.stdout: "caps": [], 2023-09-22T13:10:27.129 INFO:tasks.workunit.client.0.smithi170.stdout: "op_mask": "read, write, delete", 2023-09-22T13:10:27.129 INFO:tasks.workunit.client.0.smithi170.stdout: "default_placement": "", 2023-09-22T13:10:27.129 INFO:tasks.workunit.client.0.smithi170.stdout: "default_storage_class": "", 2023-09-22T13:10:27.129 INFO:tasks.workunit.client.0.smithi170.stdout: "placement_tags": [], 2023-09-22T13:10:27.130 INFO:tasks.workunit.client.0.smithi170.stdout: "bucket_quota": { 2023-09-22T13:10:27.130 INFO:tasks.workunit.client.0.smithi170.stdout: "enabled": false, 2023-09-22T13:10:27.130 INFO:tasks.workunit.client.0.smithi170.stdout: "check_on_raw": false, 2023-09-22T13:10:27.130 INFO:tasks.workunit.client.0.smithi170.stdout: "max_size": -1, 2023-09-22T13:10:27.130 INFO:tasks.workunit.client.0.smithi170.stdout: "max_size_kb": 0, 2023-09-22T13:10:27.130 INFO:tasks.workunit.client.0.smithi170.stdout: "max_objects": -1 2023-09-22T13:10:27.130 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:27.130 INFO:tasks.workunit.client.0.smithi170.stdout: "user_quota": { 2023-09-22T13:10:27.130 INFO:tasks.workunit.client.0.smithi170.stdout: "enabled": false, 2023-09-22T13:10:27.131 INFO:tasks.workunit.client.0.smithi170.stdout: "check_on_raw": false, 2023-09-22T13:10:27.131 INFO:tasks.workunit.client.0.smithi170.stdout: "max_size": -1, 2023-09-22T13:10:27.131 INFO:tasks.workunit.client.0.smithi170.stdout: "max_size_kb": 0, 2023-09-22T13:10:27.131 INFO:tasks.workunit.client.0.smithi170.stdout: "max_objects": -1 2023-09-22T13:10:27.131 INFO:tasks.workunit.client.0.smithi170.stdout: }, 2023-09-22T13:10:27.131 INFO:tasks.workunit.client.0.smithi170.stdout: "temp_url_keys": [], 2023-09-22T13:10:27.131 INFO:tasks.workunit.client.0.smithi170.stdout: "type": "rgw", 2023-09-22T13:10:27.131 INFO:tasks.workunit.client.0.smithi170.stdout: "mfa_ids": [] 2023-09-22T13:10:27.131 INFO:tasks.workunit.client.0.smithi170.stdout:} 2023-09-22T13:10:27.131 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:27.132 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.129+0000 7f4dfc132500 20 remove_watcher() i=0 2023-09-22T13:10:27.132 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.129+0000 7f4dfc132500 2 removed watcher, disabling cache 2023-09-22T13:10:27.133 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.130+0000 7f4dfc132500 20 remove_watcher() i=1 2023-09-22T13:10:27.133 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.131+0000 7f4dfc132500 20 remove_watcher() i=2 2023-09-22T13:10:27.133 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.132+0000 7f4dfc132500 20 remove_watcher() i=3 2023-09-22T13:10:27.133 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.133+0000 7f4dfc132500 20 remove_watcher() i=4 2023-09-22T13:10:27.133 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.134+0000 7f4dfc132500 20 remove_watcher() i=5 2023-09-22T13:10:27.135 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.135+0000 7f4dfc132500 20 remove_watcher() i=6 2023-09-22T13:10:27.136 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.136+0000 7f4dfc132500 20 remove_watcher() i=7 2023-09-22T13:10:27.136 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:27.137+0000 7f4d33fff700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:10:27.141 INFO:tasks.workunit.client.0.smithi170.stdout:phase 1.1 2023-09-22T13:10:27.156 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 24 tests from 1 test suite. 2023-09-22T13:10:27.156 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:27.156 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 24 tests from LibRGW 2023-09-22T13:10:27.156 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:10:27.167 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:27.167 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:27.742 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.INIT (586 ms) 2023-09-22T13:10:27.803 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT_NOROOT 2023-09-22T13:10:27.805 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT_NOROOT (2 ms) 2023-09-22T13:10:27.805 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:10:27.805 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT (0 ms) 2023-09-22T13:10:27.805 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SETUP_HIER1 2023-09-22T13:10:27.805 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/bar/baz/quux 2023-09-22T13:10:27.805 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/f1 2023-09-22T13:10:27.805 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/f2 2023-09-22T13:10:27.805 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/bar/f1 2023-09-22T13:10:27.805 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/bar/d1/ 2023-09-22T13:10:27.805 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/bar/baz/hungry 2023-09-22T13:10:27.806 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/bar/baz/hungry/ 2023-09-22T13:10:27.806 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/bar/baz/momma 2023-09-22T13:10:27.806 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/bar/baz/bear/ 2023-09-22T13:10:27.806 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/bar/baz/sasquatch 2023-09-22T13:10:27.807 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/bar/baz/sasquatch/ 2023-09-22T13:10:27.810 INFO:tasks.workunit.client.0.smithi170.stdout:creating: nfsroot:foo/bar/baz/frobozz 2023-09-22T13:10:27.814 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SETUP_HIER1 (70 ms) 2023-09-22T13:10:27.814 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SETUP_DIRS1 2023-09-22T13:10:27.964 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SETUP_DIRS1 (150 ms) 2023-09-22T13:10:27.964 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SETATTR 2023-09-22T13:10:27.964 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SETATTR (0 ms) 2023-09-22T13:10:27.964 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_CREATE_DIRS1 2023-09-22T13:10:27.986 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_CREATE_DIRS1 (21 ms) 2023-09-22T13:10:27.986 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_SETUP_RENAME1 2023-09-22T13:10:28.038 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_SETUP_RENAME1 (53 ms) 2023-09-22T13:10:28.039 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_INTRABUCKET_RENAME1 2023-09-22T13:10:28.039 INFO:tasks.workunit.client.0.smithi170.stdout:rename file rfile_0 to rfile_r0 (bucket brename0) 2023-09-22T13:10:28.045 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_INTRABUCKET_RENAME1 (7 ms) 2023-09-22T13:10:28.045 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_CROSSBUCKET_RENAME1 2023-09-22T13:10:28.045 INFO:tasks.workunit.client.0.smithi170.stdout:rename file rfile_1 (bucket brename0) to rfile_rhilldog (bucket brename1) 2023-09-22T13:10:28.052 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_CROSSBUCKET_RENAME1 (7 ms) 2023-09-22T13:10:28.052 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.GETATTR_DIRS1 2023-09-22T13:10:28.052 INFO:tasks.workunit.client.0.smithi170.stdout:scanning objects in /bdirs1/dir_0 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_0/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_0/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_0/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_0/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout:scanning objects in /bdirs1/dir_1 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_1/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_1/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_1/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_1/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout:scanning objects in /bdirs1/dir_2 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_2/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.053 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_2/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:10:28.054 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_2/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.054 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_2/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:10:28.054 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.GETATTR_DIRS1 (0 ms) 2023-09-22T13:10:28.054 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READ_DIRS1 2023-09-22T13:10:28.054 INFO:tasks.workunit.client.0.smithi170.stdout:read back objects in /bdirs1/dir_0 2023-09-22T13:10:28.054 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_0/sfile_0 2023-09-22T13:10:28.054 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:10:28.054 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_0/sfile_1 2023-09-22T13:10:28.055 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:10:28.055 INFO:tasks.workunit.client.0.smithi170.stdout:read back objects in /bdirs1/dir_1 2023-09-22T13:10:28.055 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_1/sfile_0 2023-09-22T13:10:28.056 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:10:28.056 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_1/sfile_1 2023-09-22T13:10:28.058 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:10:28.059 INFO:tasks.workunit.client.0.smithi170.stdout:read back objects in /bdirs1/dir_2 2023-09-22T13:10:28.059 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_2/sfile_0 2023-09-22T13:10:28.060 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:10:28.060 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_2/sfile_1 2023-09-22T13:10:28.062 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:10:28.062 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READ_DIRS1 (10 ms) 2023-09-22T13:10:28.062 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READF_SETUP1 2023-09-22T13:10:28.239 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READF_SETUP1 (177 ms) 2023-09-22T13:10:28.240 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READF_DIRS1 2023-09-22T13:10:28.240 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READF_DIRS1 (0 ms) 2023-09-22T13:10:28.240 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.WRITEF_DIRS1 2023-09-22T13:10:28.240 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.WRITEF_DIRS1 (0 ms) 2023-09-22T13:10:28.240 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RELEASE_DIRS1 2023-09-22T13:10:28.240 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:10:28.240 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:10:28.240 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:10:28.240 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:10:28.240 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:10:28.241 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:10:28.241 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:10:28.241 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:10:28.241 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:10:28.241 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:10:28.241 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:10:28.241 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:10:28.241 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RELEASE_DIRS1 (0 ms) 2023-09-22T13:10:28.241 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.HIER1 2023-09-22T13:10:28.248 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/f2 (f2): FILE 2023-09-22T13:10:28.249 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/f1 (f1): FILE 2023-09-22T13:10:28.253 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/f1 (f1): FILE 2023-09-22T13:10:28.255 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/d1 (d1): DIR 2023-09-22T13:10:28.260 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:10:28.260 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:10:28.261 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/quux (quux): FILE 2023-09-22T13:10:28.262 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/momma (momma): FILE 2023-09-22T13:10:28.262 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:10:28.263 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:10:28.264 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/frobozz (frobozz): FILE 2023-09-22T13:10:28.266 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/bear (bear): DIR 2023-09-22T13:10:28.267 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz (baz): DIR 2023-09-22T13:10:28.267 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar (bar): DIR 2023-09-22T13:10:28.267 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo (foo): DIR 2023-09-22T13:10:28.267 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot (nfsroot): DIR 2023-09-22T13:10:28.267 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.HIER1 (27 ms) 2023-09-22T13:10:28.267 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:10:28.267 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (0 ms) 2023-09-22T13:10:28.267 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:10:28.267 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:10:28.268 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:10:28.268 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:10:28.268 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:10:28.268 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:10:28.268 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:10:28.268 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:10:28.268 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:10:28.268 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:10:28.268 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:10:28.269 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:28.266+0000 7ff06944a540 -1 shutting down 2023-09-22T13:10:28.269 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:28.266+0000 7fef7d7fa700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156031, si_uid : 1000, si_addr0x3e80002617f, si_status0 2023-09-22T13:10:28.344 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SHUTDOWN (77 ms) 2023-09-22T13:10:28.344 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 24 tests from LibRGW (1187 ms total) 2023-09-22T13:10:28.344 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:28.344 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:28.344 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 24 tests from 1 test suite ran. (1187 ms total) 2023-09-22T13:10:28.344 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 24 tests. 2023-09-22T13:10:28.346 INFO:tasks.workunit.client.0.smithi170.stdout:phase 1.2 2023-09-22T13:10:28.361 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 24 tests from 1 test suite. 2023-09-22T13:10:28.361 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:28.361 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 24 tests from LibRGW 2023-09-22T13:10:28.361 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:10:28.369 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:28.369 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:28.463 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.INIT (102 ms) 2023-09-22T13:10:28.463 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT_NOROOT 2023-09-22T13:10:28.465 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT_NOROOT (1 ms) 2023-09-22T13:10:28.465 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:10:28.465 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT (0 ms) 2023-09-22T13:10:28.465 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SETUP_HIER1 2023-09-22T13:10:28.465 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SETUP_HIER1 (0 ms) 2023-09-22T13:10:28.465 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SETUP_DIRS1 2023-09-22T13:10:28.465 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SETUP_DIRS1 (0 ms) 2023-09-22T13:10:28.465 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SETATTR 2023-09-22T13:10:28.465 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SETATTR (0 ms) 2023-09-22T13:10:28.465 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_CREATE_DIRS1 2023-09-22T13:10:28.465 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_CREATE_DIRS1 (0 ms) 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_SETUP_RENAME1 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_SETUP_RENAME1 (0 ms) 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_INTRABUCKET_RENAME1 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_INTRABUCKET_RENAME1 (0 ms) 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_CROSSBUCKET_RENAME1 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_CROSSBUCKET_RENAME1 (0 ms) 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.GETATTR_DIRS1 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.GETATTR_DIRS1 (0 ms) 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READ_DIRS1 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READ_DIRS1 (0 ms) 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READF_SETUP1 2023-09-22T13:10:28.466 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READF_SETUP1 (0 ms) 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READF_DIRS1 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READF_DIRS1 (0 ms) 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.WRITEF_DIRS1 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.WRITEF_DIRS1 (0 ms) 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RELEASE_DIRS1 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RELEASE_DIRS1 (0 ms) 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.HIER1 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.HIER1 (0 ms) 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (0 ms) 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:10:28.467 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:10:28.468 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:10:28.468 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:10:28.468 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:10:28.468 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:10:28.468 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:10:28.468 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:10:28.468 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:10:28.468 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:10:28.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:28.464+0000 7f8e33e6e540 -1 shutting down 2023-09-22T13:10:28.468 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:28.464+0000 7f8d3b7fe700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156115, si_uid : 1000, si_addr0x3e8000261d3, si_status0 2023-09-22T13:10:28.564 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SHUTDOWN (99 ms) 2023-09-22T13:10:28.564 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 24 tests from LibRGW (202 ms total) 2023-09-22T13:10:28.564 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:28.564 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:28.564 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 24 tests from 1 test suite ran. (202 ms total) 2023-09-22T13:10:28.564 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 24 tests. 2023-09-22T13:10:28.566 INFO:tasks.workunit.client.0.smithi170.stdout:phase 1.3 2023-09-22T13:10:28.581 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 24 tests from 1 test suite. 2023-09-22T13:10:28.581 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:28.581 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 24 tests from LibRGW 2023-09-22T13:10:28.581 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:10:28.591 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:28.591 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:28.689 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.INIT (107 ms) 2023-09-22T13:10:28.689 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT_NOROOT 2023-09-22T13:10:28.690 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT_NOROOT (2 ms) 2023-09-22T13:10:28.690 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:10:28.690 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT (0 ms) 2023-09-22T13:10:28.691 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SETUP_HIER1 2023-09-22T13:10:28.696 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SETUP_HIER1 (6 ms) 2023-09-22T13:10:28.697 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SETUP_DIRS1 2023-09-22T13:10:28.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SETUP_DIRS1 (78 ms) 2023-09-22T13:10:28.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SETATTR 2023-09-22T13:10:28.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SETATTR (0 ms) 2023-09-22T13:10:28.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_CREATE_DIRS1 2023-09-22T13:10:28.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_CREATE_DIRS1 (0 ms) 2023-09-22T13:10:28.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_SETUP_RENAME1 2023-09-22T13:10:28.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_SETUP_RENAME1 (0 ms) 2023-09-22T13:10:28.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_INTRABUCKET_RENAME1 2023-09-22T13:10:28.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_INTRABUCKET_RENAME1 (0 ms) 2023-09-22T13:10:28.775 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RGW_CROSSBUCKET_RENAME1 2023-09-22T13:10:28.776 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RGW_CROSSBUCKET_RENAME1 (0 ms) 2023-09-22T13:10:28.776 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.GETATTR_DIRS1 2023-09-22T13:10:28.776 INFO:tasks.workunit.client.0.smithi170.stdout:scanning objects in /bdirs1/dir_0 2023-09-22T13:10:28.776 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_0/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.776 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_0/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:10:28.776 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_0/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.776 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_0/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:10:28.776 INFO:tasks.workunit.client.0.smithi170.stdout:scanning objects in /bdirs1/dir_1 2023-09-22T13:10:28.776 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_1/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.776 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_1/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:10:28.776 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_1/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.777 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_1/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:10:28.777 INFO:tasks.workunit.client.0.smithi170.stdout:scanning objects in /bdirs1/dir_2 2023-09-22T13:10:28.777 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_2/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.777 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_2/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:10:28.777 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_2/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:10:28.777 INFO:tasks.workunit.client.0.smithi170.stdout: /bdirs1/dir_2/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:10:28.777 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.GETATTR_DIRS1 (0 ms) 2023-09-22T13:10:28.777 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READ_DIRS1 2023-09-22T13:10:28.777 INFO:tasks.workunit.client.0.smithi170.stdout:read back objects in /bdirs1/dir_0 2023-09-22T13:10:28.777 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_0/sfile_0 2023-09-22T13:10:28.778 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:10:28.778 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_0/sfile_1 2023-09-22T13:10:28.778 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:10:28.778 INFO:tasks.workunit.client.0.smithi170.stdout:read back objects in /bdirs1/dir_1 2023-09-22T13:10:28.778 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_1/sfile_0 2023-09-22T13:10:28.779 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:10:28.779 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_1/sfile_1 2023-09-22T13:10:28.781 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:10:28.781 INFO:tasks.workunit.client.0.smithi170.stdout:read back objects in /bdirs1/dir_2 2023-09-22T13:10:28.781 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_2/sfile_0 2023-09-22T13:10:28.782 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:10:28.782 INFO:tasks.workunit.client.0.smithi170.stdout:reading 0,256 dir_2/sfile_1 2023-09-22T13:10:28.784 INFO:tasks.workunit.client.0.smithi170.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:10:28.784 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READ_DIRS1 (10 ms) 2023-09-22T13:10:28.785 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READF_SETUP1 2023-09-22T13:10:28.785 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READF_SETUP1 (0 ms) 2023-09-22T13:10:28.785 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READF_DIRS1 2023-09-22T13:10:28.785 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READF_DIRS1 (0 ms) 2023-09-22T13:10:28.785 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.WRITEF_DIRS1 2023-09-22T13:10:28.785 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.WRITEF_DIRS1 (0 ms) 2023-09-22T13:10:28.785 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.RELEASE_DIRS1 2023-09-22T13:10:28.785 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:10:28.785 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:10:28.785 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:10:28.785 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:10:28.786 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:10:28.786 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:10:28.786 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:10:28.786 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:10:28.786 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:10:28.786 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:10:28.786 INFO:tasks.workunit.client.0.smithi170.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:10:28.786 INFO:tasks.workunit.client.0.smithi170.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:10:28.786 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.RELEASE_DIRS1 (0 ms) 2023-09-22T13:10:28.786 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.HIER1 2023-09-22T13:10:28.798 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/f2 (f2): FILE 2023-09-22T13:10:28.798 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/f1 (f1): FILE 2023-09-22T13:10:28.803 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/f1 (f1): FILE 2023-09-22T13:10:28.805 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/d1 (d1): DIR 2023-09-22T13:10:28.810 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:10:28.810 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:10:28.811 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/quux (quux): FILE 2023-09-22T13:10:28.812 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/momma (momma): FILE 2023-09-22T13:10:28.812 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:10:28.813 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:10:28.814 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/frobozz (frobozz): FILE 2023-09-22T13:10:28.817 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz/bear (bear): DIR 2023-09-22T13:10:28.817 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar/baz (baz): DIR 2023-09-22T13:10:28.817 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo/bar (bar): DIR 2023-09-22T13:10:28.817 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot/foo (foo): DIR 2023-09-22T13:10:28.817 INFO:tasks.workunit.client.0.smithi170.stdout:/nfsroot (nfsroot): DIR 2023-09-22T13:10:28.817 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.HIER1 (32 ms) 2023-09-22T13:10:28.817 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (0 ms) 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:10:28.818 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:10:28.819 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:28.816+0000 7f372373a540 -1 shutting down 2023-09-22T13:10:28.819 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:28.816+0000 7f362affd700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156199, si_uid : 1000, si_addr0x3e800026227, si_status0 2023-09-22T13:10:28.890 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SHUTDOWN (74 ms) 2023-09-22T13:10:28.890 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 24 tests from LibRGW (309 ms total) 2023-09-22T13:10:28.890 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:28.890 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:28.890 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 24 tests from 1 test suite ran. (309 ms total) 2023-09-22T13:10:28.890 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 24 tests. 2023-09-22T13:10:28.892 INFO:tasks.workunit.client.0.smithi170.stdout:phase 2.1 2023-09-22T13:10:28.908 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 9 tests from 1 test suite. 2023-09-22T13:10:28.908 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:28.908 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 9 tests from LibRGW 2023-09-22T13:10:28.908 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:10:28.916 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:28.916 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:29.012 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.INIT (105 ms) 2023-09-22T13:10:29.012 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:10:29.013 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:10:29.013 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:10:29.022 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CREATE_BUCKET (9 ms) 2023-09-22T13:10:29.022 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:10:29.022 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_BUCKET (0 ms) 2023-09-22T13:10:29.022 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CREATE_BUCKET_MULTI 2023-09-22T13:10:29.031 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave0 2023-09-22T13:10:29.042 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave1 2023-09-22T13:10:29.050 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave2 2023-09-22T13:10:29.059 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave3 2023-09-22T13:10:29.068 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave4 2023-09-22T13:10:29.078 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave5 2023-09-22T13:10:29.088 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave6 2023-09-22T13:10:29.098 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave7 2023-09-22T13:10:29.108 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave8 2023-09-22T13:10:29.118 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave9 2023-09-22T13:10:29.119 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CREATE_BUCKET_MULTI (96 ms) 2023-09-22T13:10:29.119 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_BUCKET_MULTI 2023-09-22T13:10:29.454 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_BUCKET_MULTI (336 ms) 2023-09-22T13:10:29.454 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:10:29.454 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:10:29.455 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:10:29.455 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:10:29.455 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:10:29.455 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:29.453+0000 7f91a1ff7540 -1 shutting down 2023-09-22T13:10:29.455 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:29.453+0000 7f90b9ffb700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156283, si_uid : 1000, si_addr0x3e80002627b, si_status0 2023-09-22T13:10:29.514 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SHUTDOWN (60 ms) 2023-09-22T13:10:29.514 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 9 tests from LibRGW (607 ms total) 2023-09-22T13:10:29.514 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:29.515 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:29.515 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 9 tests from 1 test suite ran. (607 ms total) 2023-09-22T13:10:29.515 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 9 tests. 2023-09-22T13:10:29.516 INFO:tasks.workunit.client.0.smithi170.stdout:phase 2.2 2023-09-22T13:10:29.533 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 9 tests from 1 test suite. 2023-09-22T13:10:29.533 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:29.533 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 9 tests from LibRGW 2023-09-22T13:10:29.533 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:10:29.541 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:29.541 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:29.641 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.INIT (109 ms) 2023-09-22T13:10:29.641 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:10:29.642 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:10:29.642 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:10:29.643 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CREATE_BUCKET (0 ms) 2023-09-22T13:10:29.643 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:10:29.679 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_BUCKET (37 ms) 2023-09-22T13:10:29.679 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CREATE_BUCKET_MULTI 2023-09-22T13:10:29.687 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave0 2023-09-22T13:10:29.698 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave1 2023-09-22T13:10:29.708 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave2 2023-09-22T13:10:29.717 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave3 2023-09-22T13:10:29.729 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave4 2023-09-22T13:10:29.739 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave5 2023-09-22T13:10:29.751 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave6 2023-09-22T13:10:29.763 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave7 2023-09-22T13:10:29.772 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave8 2023-09-22T13:10:29.780 INFO:tasks.workunit.client.0.smithi170.stdout:created: sorrydave9 2023-09-22T13:10:29.780 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CREATE_BUCKET_MULTI (100 ms) 2023-09-22T13:10:29.780 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_BUCKET_MULTI 2023-09-22T13:10:30.333 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_BUCKET_MULTI (553 ms) 2023-09-22T13:10:30.333 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:10:30.333 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:10:30.333 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:10:30.333 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:10:30.333 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:10:30.334 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:30.332+0000 7f5e9f4e9540 -1 shutting down 2023-09-22T13:10:30.334 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:30.332+0000 7f5da6ffd700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156367, si_uid : 1000, si_addr0x3e8000262cf, si_status0 2023-09-22T13:10:30.443 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SHUTDOWN (110 ms) 2023-09-22T13:10:30.443 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 9 tests from LibRGW (910 ms total) 2023-09-22T13:10:30.443 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:30.443 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:30.443 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 9 tests from 1 test suite ran. (910 ms total) 2023-09-22T13:10:30.443 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 9 tests. 2023-09-22T13:10:30.445 INFO:tasks.workunit.client.0.smithi170.stdout:phase 3.1 2023-09-22T13:10:31.221 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 13 tests from 1 test suite. 2023-09-22T13:10:31.221 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:31.221 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 13 tests from LibRGW 2023-09-22T13:10:31.221 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:10:31.230 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:31.230 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:31.329 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.INIT (108 ms) 2023-09-22T13:10:31.329 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:10:31.330 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT (2 ms) 2023-09-22T13:10:31.330 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:10:31.339 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CREATE_BUCKET (9 ms) 2023-09-22T13:10:31.339 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:10:31.343 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (3 ms) 2023-09-22T13:10:31.343 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:10:31.345 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (3 ms) 2023-09-22T13:10:31.345 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LARGE1 2023-09-22T13:10:31.796 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LARGE1 (451 ms) 2023-09-22T13:10:31.797 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LARGE2 2023-09-22T13:10:31.860 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LARGE2 (64 ms) 2023-09-22T13:10:31.860 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:10:31.861 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:10:31.861 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:10:31.861 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_OBJECT (0 ms) 2023-09-22T13:10:31.861 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:10:31.861 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_BUCKET (0 ms) 2023-09-22T13:10:31.861 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:10:31.861 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:10:31.861 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:10:31.861 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:10:31.861 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:10:31.862 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:31.860+0000 7fa4dd029540 -1 shutting down 2023-09-22T13:10:31.862 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:31.860+0000 7fa3f1ffb700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156451, si_uid : 1000, si_addr0x3e800026323, si_status0 2023-09-22T13:10:32.030 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SHUTDOWN (170 ms) 2023-09-22T13:10:32.030 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 13 tests from LibRGW (810 ms total) 2023-09-22T13:10:32.030 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:32.030 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:32.031 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 13 tests from 1 test suite ran. (810 ms total) 2023-09-22T13:10:32.031 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 13 tests. 2023-09-22T13:10:32.034 INFO:tasks.workunit.client.0.smithi170.stdout:phase 3.2 2023-09-22T13:10:32.778 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 13 tests from 1 test suite. 2023-09-22T13:10:32.778 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:32.778 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 13 tests from LibRGW 2023-09-22T13:10:32.778 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:10:32.786 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:32.786 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:32.888 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.INIT (110 ms) 2023-09-22T13:10:32.888 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:10:32.889 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:10:32.890 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:10:32.890 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CREATE_BUCKET (0 ms) 2023-09-22T13:10:32.890 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:10:32.895 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (6 ms) 2023-09-22T13:10:32.896 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:10:32.902 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (6 ms) 2023-09-22T13:10:32.902 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LARGE1 2023-09-22T13:10:32.986 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LARGE1 (84 ms) 2023-09-22T13:10:32.986 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LARGE2 2023-09-22T13:10:32.986 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LARGE2 (0 ms) 2023-09-22T13:10:32.986 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:10:32.986 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:10:32.986 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:10:32.994 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_OBJECT (9 ms) 2023-09-22T13:10:32.995 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:10:33.030 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_BUCKET (35 ms) 2023-09-22T13:10:33.030 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:10:33.030 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:10:33.030 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:10:33.030 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:10:33.031 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:10:33.031 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:33.029+0000 7f9bf6b57540 -1 shutting down 2023-09-22T13:10:33.031 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:33.029+0000 7f9b08ff9700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156535, si_uid : 1000, si_addr0x3e800026377, si_status0 2023-09-22T13:10:33.089 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SHUTDOWN (59 ms) 2023-09-22T13:10:33.089 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 13 tests from LibRGW (310 ms total) 2023-09-22T13:10:33.089 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:33.089 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:33.089 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 13 tests from 1 test suite ran. (310 ms total) 2023-09-22T13:10:33.089 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 13 tests. 2023-09-22T13:10:33.092 INFO:tasks.workunit.client.0.smithi170.stdout:phase 4.1 2023-09-22T13:10:33.107 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 11 tests from 1 test suite. 2023-09-22T13:10:33.107 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:33.107 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 11 tests from LibRGW 2023-09-22T13:10:33.107 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.TVAR 2023-09-22T13:10:33.108 INFO:tasks.workunit.client.0.smithi170.stdout:read i1: 64001 2023-09-22T13:10:33.108 INFO:tasks.workunit.client.0.smithi170.stdout:read s1: blunderbuss 2023-09-22T13:10:33.108 INFO:tasks.workunit.client.0.smithi170.stdout:read s3: [ OK ] LibRGW.TVAR (0 ms) 2023-09-22T13:10:33.108 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:10:33.118 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:33.118 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:33.200 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.INIT (93 ms) 2023-09-22T13:10:33.201 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:10:33.202 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT (2 ms) 2023-09-22T13:10:33.202 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:10:33.219 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (17 ms) 2023-09-22T13:10:33.219 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:10:33.833 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_0 2023-09-22T13:10:33.834 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_1 2023-09-22T13:10:33.834 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_2 2023-09-22T13:10:33.834 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_3 2023-09-22T13:10:33.834 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_4 2023-09-22T13:10:33.834 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_5 2023-09-22T13:10:33.834 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_6 2023-09-22T13:10:33.834 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_7 2023-09-22T13:10:33.834 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_8 2023-09-22T13:10:33.835 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_9 2023-09-22T13:10:33.835 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_10 2023-09-22T13:10:33.835 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_11 2023-09-22T13:10:33.835 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_12 2023-09-22T13:10:33.835 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_13 2023-09-22T13:10:33.835 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_14 2023-09-22T13:10:33.835 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_15 2023-09-22T13:10:33.835 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_16 2023-09-22T13:10:33.835 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_17 2023-09-22T13:10:33.835 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_18 2023-09-22T13:10:33.836 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_19 2023-09-22T13:10:33.836 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_20 2023-09-22T13:10:33.836 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_21 2023-09-22T13:10:33.836 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_22 2023-09-22T13:10:33.836 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_23 2023-09-22T13:10:33.836 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_24 2023-09-22T13:10:33.836 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_25 2023-09-22T13:10:33.836 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_26 2023-09-22T13:10:33.836 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_27 2023-09-22T13:10:33.836 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_28 2023-09-22T13:10:33.837 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_29 2023-09-22T13:10:33.837 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_30 2023-09-22T13:10:33.837 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_31 2023-09-22T13:10:33.837 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_32 2023-09-22T13:10:33.837 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_33 2023-09-22T13:10:33.837 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_34 2023-09-22T13:10:33.837 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_35 2023-09-22T13:10:33.837 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_36 2023-09-22T13:10:33.837 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_37 2023-09-22T13:10:33.837 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_38 2023-09-22T13:10:33.837 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_39 2023-09-22T13:10:33.838 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_40 2023-09-22T13:10:33.838 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_41 2023-09-22T13:10:33.838 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_42 2023-09-22T13:10:33.838 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_43 2023-09-22T13:10:33.838 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_44 2023-09-22T13:10:33.838 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_45 2023-09-22T13:10:33.838 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_46 2023-09-22T13:10:33.838 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_47 2023-09-22T13:10:33.838 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_48 2023-09-22T13:10:33.838 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_49 2023-09-22T13:10:33.839 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_50 2023-09-22T13:10:33.839 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_51 2023-09-22T13:10:33.839 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_52 2023-09-22T13:10:33.839 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_53 2023-09-22T13:10:33.839 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_54 2023-09-22T13:10:33.839 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_55 2023-09-22T13:10:33.839 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_56 2023-09-22T13:10:33.839 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_57 2023-09-22T13:10:33.839 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_58 2023-09-22T13:10:33.839 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_59 2023-09-22T13:10:33.840 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_60 2023-09-22T13:10:33.840 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_61 2023-09-22T13:10:33.840 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_62 2023-09-22T13:10:33.840 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_63 2023-09-22T13:10:33.840 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_64 2023-09-22T13:10:33.840 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_65 2023-09-22T13:10:33.840 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_66 2023-09-22T13:10:33.840 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_67 2023-09-22T13:10:33.840 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_68 2023-09-22T13:10:33.840 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_69 2023-09-22T13:10:33.840 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_70 2023-09-22T13:10:33.841 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_71 2023-09-22T13:10:33.841 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_72 2023-09-22T13:10:33.841 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_73 2023-09-22T13:10:33.841 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_74 2023-09-22T13:10:33.841 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_75 2023-09-22T13:10:33.841 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_76 2023-09-22T13:10:33.841 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_77 2023-09-22T13:10:33.841 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_78 2023-09-22T13:10:33.841 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_79 2023-09-22T13:10:33.841 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_80 2023-09-22T13:10:33.841 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_81 2023-09-22T13:10:33.842 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_82 2023-09-22T13:10:33.842 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_83 2023-09-22T13:10:33.842 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_84 2023-09-22T13:10:33.842 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_85 2023-09-22T13:10:33.842 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_86 2023-09-22T13:10:33.842 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_87 2023-09-22T13:10:33.842 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_88 2023-09-22T13:10:33.842 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_89 2023-09-22T13:10:33.842 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_90 2023-09-22T13:10:33.842 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_91 2023-09-22T13:10:33.843 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_92 2023-09-22T13:10:33.843 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_93 2023-09-22T13:10:33.843 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_94 2023-09-22T13:10:33.843 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_95 2023-09-22T13:10:33.843 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_96 2023-09-22T13:10:33.843 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_97 2023-09-22T13:10:33.843 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_98 2023-09-22T13:10:33.843 INFO:tasks.workunit.client.0.smithi170.stdout:created: dmarker:f_99 2023-09-22T13:10:33.843 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (614 ms) 2023-09-22T13:10:33.843 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:10:33.845 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=. flags=4 2023-09-22T13:10:33.845 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=.. flags=4 2023-09-22T13:10:33.845 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_0 flags=8 2023-09-22T13:10:33.845 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_1 flags=8 2023-09-22T13:10:33.845 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_10 flags=8 2023-09-22T13:10:33.845 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_11 flags=8 2023-09-22T13:10:33.845 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_12 flags=8 2023-09-22T13:10:33.845 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_13 flags=8 2023-09-22T13:10:33.846 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_14 flags=8 2023-09-22T13:10:33.846 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_15 flags=8 2023-09-22T13:10:33.846 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_16 flags=8 2023-09-22T13:10:33.846 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_17 flags=8 2023-09-22T13:10:33.846 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_18 flags=8 2023-09-22T13:10:33.846 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_19 flags=8 2023-09-22T13:10:33.846 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_2 flags=8 2023-09-22T13:10:33.846 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_20 flags=8 2023-09-22T13:10:33.846 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_21 flags=8 2023-09-22T13:10:33.846 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_22 flags=8 2023-09-22T13:10:33.847 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_23 flags=8 2023-09-22T13:10:33.847 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_24 flags=8 2023-09-22T13:10:33.847 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_25 flags=8 2023-09-22T13:10:33.847 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_26 flags=8 2023-09-22T13:10:33.847 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_27 flags=8 2023-09-22T13:10:33.847 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_28 flags=8 2023-09-22T13:10:33.847 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_29 flags=8 2023-09-22T13:10:33.847 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_3 flags=8 2023-09-22T13:10:33.847 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_30 flags=8 2023-09-22T13:10:33.847 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_31 flags=8 2023-09-22T13:10:33.848 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_32 flags=8 2023-09-22T13:10:33.848 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_33 flags=8 2023-09-22T13:10:33.848 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_34 flags=8 2023-09-22T13:10:33.848 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_35 flags=8 2023-09-22T13:10:33.848 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_36 flags=8 2023-09-22T13:10:33.848 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_37 flags=8 2023-09-22T13:10:33.848 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_38 flags=8 2023-09-22T13:10:33.848 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_39 flags=8 2023-09-22T13:10:33.848 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_4 flags=8 2023-09-22T13:10:33.848 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_40 flags=8 2023-09-22T13:10:33.849 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_41 flags=8 2023-09-22T13:10:33.849 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_42 flags=8 2023-09-22T13:10:33.849 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_43 flags=8 2023-09-22T13:10:33.849 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_44 flags=8 2023-09-22T13:10:33.849 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_45 flags=8 2023-09-22T13:10:33.849 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_46 flags=8 2023-09-22T13:10:33.849 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_47 flags=8 2023-09-22T13:10:33.849 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_48 flags=8 2023-09-22T13:10:33.849 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_49 flags=8 2023-09-22T13:10:33.849 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_5 flags=8 2023-09-22T13:10:33.849 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_50 flags=8 2023-09-22T13:10:33.850 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_51 flags=8 2023-09-22T13:10:33.850 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_52 flags=8 2023-09-22T13:10:33.850 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_53 flags=8 2023-09-22T13:10:33.850 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_54 flags=8 2023-09-22T13:10:33.850 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_55 flags=8 2023-09-22T13:10:33.850 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_56 flags=8 2023-09-22T13:10:33.850 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_57 flags=8 2023-09-22T13:10:33.850 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_58 flags=8 2023-09-22T13:10:33.850 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_59 flags=8 2023-09-22T13:10:33.850 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_6 flags=8 2023-09-22T13:10:33.851 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_60 flags=8 2023-09-22T13:10:33.851 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_61 flags=8 2023-09-22T13:10:33.851 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_62 flags=8 2023-09-22T13:10:33.851 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_63 flags=8 2023-09-22T13:10:33.851 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_64 flags=8 2023-09-22T13:10:33.851 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_65 flags=8 2023-09-22T13:10:33.851 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_66 flags=8 2023-09-22T13:10:33.851 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_67 flags=8 2023-09-22T13:10:33.851 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_68 flags=8 2023-09-22T13:10:33.851 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_69 flags=8 2023-09-22T13:10:33.851 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_7 flags=8 2023-09-22T13:10:33.852 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_70 flags=8 2023-09-22T13:10:33.852 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_71 flags=8 2023-09-22T13:10:33.852 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_72 flags=8 2023-09-22T13:10:33.852 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_73 flags=8 2023-09-22T13:10:33.852 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_74 flags=8 2023-09-22T13:10:33.852 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_75 flags=8 2023-09-22T13:10:33.852 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_76 flags=8 2023-09-22T13:10:33.852 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_77 flags=8 2023-09-22T13:10:33.852 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_78 flags=8 2023-09-22T13:10:33.852 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_79 flags=8 2023-09-22T13:10:33.852 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_8 flags=8 2023-09-22T13:10:33.853 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_80 flags=8 2023-09-22T13:10:33.853 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_81 flags=8 2023-09-22T13:10:33.853 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_82 flags=8 2023-09-22T13:10:33.853 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_83 flags=8 2023-09-22T13:10:33.853 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_84 flags=8 2023-09-22T13:10:33.853 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_85 flags=8 2023-09-22T13:10:33.853 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_86 flags=8 2023-09-22T13:10:33.853 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_87 flags=8 2023-09-22T13:10:33.853 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_88 flags=8 2023-09-22T13:10:33.853 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_89 flags=8 2023-09-22T13:10:33.854 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_9 flags=8 2023-09-22T13:10:33.854 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_90 flags=8 2023-09-22T13:10:33.854 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_91 flags=8 2023-09-22T13:10:33.854 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_92 flags=8 2023-09-22T13:10:33.854 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_93 flags=8 2023-09-22T13:10:33.854 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_94 flags=8 2023-09-22T13:10:33.854 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_95 flags=8 2023-09-22T13:10:33.854 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_96 flags=8 2023-09-22T13:10:33.854 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_97 flags=8 2023-09-22T13:10:33.855 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_98 flags=8 2023-09-22T13:10:33.855 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_99 flags=8 2023-09-22T13:10:33.855 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_READDIR (2 ms) 2023-09-22T13:10:33.855 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER2_READDIR 2023-09-22T13:10:33.855 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_0 flags=8 2023-09-22T13:10:33.855 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_1 flags=8 2023-09-22T13:10:33.855 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_10 flags=8 2023-09-22T13:10:33.855 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_11 flags=8 2023-09-22T13:10:33.855 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_12 flags=8 2023-09-22T13:10:33.855 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_13 flags=8 2023-09-22T13:10:33.855 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_14 flags=8 2023-09-22T13:10:33.856 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_15 flags=8 2023-09-22T13:10:33.856 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_16 flags=8 2023-09-22T13:10:33.856 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_17 flags=8 2023-09-22T13:10:33.856 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_18 flags=8 2023-09-22T13:10:33.856 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_19 flags=8 2023-09-22T13:10:33.856 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_2 flags=8 2023-09-22T13:10:33.856 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_20 flags=8 2023-09-22T13:10:33.856 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_21 flags=8 2023-09-22T13:10:33.856 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_22 flags=8 2023-09-22T13:10:33.856 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_23 flags=8 2023-09-22T13:10:33.857 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_24 flags=8 2023-09-22T13:10:33.857 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_25 flags=8 2023-09-22T13:10:33.857 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_26 flags=8 2023-09-22T13:10:33.857 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_27 flags=8 2023-09-22T13:10:33.857 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_28 flags=8 2023-09-22T13:10:33.857 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_29 flags=8 2023-09-22T13:10:33.857 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_3 flags=8 2023-09-22T13:10:33.857 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_30 flags=8 2023-09-22T13:10:33.857 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_31 flags=8 2023-09-22T13:10:33.857 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_32 flags=8 2023-09-22T13:10:33.857 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_33 flags=8 2023-09-22T13:10:33.858 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_34 flags=8 2023-09-22T13:10:33.858 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_35 flags=8 2023-09-22T13:10:33.858 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_36 flags=8 2023-09-22T13:10:33.858 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_37 flags=8 2023-09-22T13:10:33.858 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_38 flags=8 2023-09-22T13:10:33.858 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_39 flags=8 2023-09-22T13:10:33.858 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_4 flags=8 2023-09-22T13:10:33.858 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_40 flags=8 2023-09-22T13:10:33.858 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_41 flags=8 2023-09-22T13:10:33.858 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_42 flags=8 2023-09-22T13:10:33.858 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_43 flags=8 2023-09-22T13:10:33.859 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_44 flags=8 2023-09-22T13:10:33.859 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_45 flags=8 2023-09-22T13:10:33.859 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_46 flags=8 2023-09-22T13:10:33.859 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_47 flags=8 2023-09-22T13:10:33.859 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_48 flags=8 2023-09-22T13:10:33.859 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_49 flags=8 2023-09-22T13:10:33.859 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_5 flags=8 2023-09-22T13:10:33.859 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_50 flags=8 2023-09-22T13:10:33.859 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_51 flags=8 2023-09-22T13:10:33.859 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_52 flags=8 2023-09-22T13:10:33.859 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_53 flags=8 2023-09-22T13:10:33.860 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_54 flags=8 2023-09-22T13:10:33.860 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_55 flags=8 2023-09-22T13:10:33.860 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_56 flags=8 2023-09-22T13:10:33.860 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_57 flags=8 2023-09-22T13:10:33.860 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_58 flags=8 2023-09-22T13:10:33.860 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_59 flags=8 2023-09-22T13:10:33.860 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_6 flags=8 2023-09-22T13:10:33.860 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_60 flags=8 2023-09-22T13:10:33.860 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_61 flags=8 2023-09-22T13:10:33.860 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_62 flags=8 2023-09-22T13:10:33.860 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_63 flags=8 2023-09-22T13:10:33.861 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_64 flags=8 2023-09-22T13:10:33.861 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_65 flags=8 2023-09-22T13:10:33.861 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_66 flags=8 2023-09-22T13:10:33.861 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_67 flags=8 2023-09-22T13:10:33.861 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_68 flags=8 2023-09-22T13:10:33.861 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_69 flags=8 2023-09-22T13:10:33.861 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_7 flags=8 2023-09-22T13:10:33.861 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_70 flags=8 2023-09-22T13:10:33.861 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_71 flags=8 2023-09-22T13:10:33.861 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_72 flags=8 2023-09-22T13:10:33.861 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_73 flags=8 2023-09-22T13:10:33.862 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_74 flags=8 2023-09-22T13:10:33.862 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_75 flags=8 2023-09-22T13:10:33.862 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_76 flags=8 2023-09-22T13:10:33.862 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_77 flags=8 2023-09-22T13:10:33.862 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_78 flags=8 2023-09-22T13:10:33.862 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_79 flags=8 2023-09-22T13:10:33.862 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_8 flags=8 2023-09-22T13:10:33.862 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_80 flags=8 2023-09-22T13:10:33.862 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_81 flags=8 2023-09-22T13:10:33.862 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_82 flags=8 2023-09-22T13:10:33.863 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_83 flags=8 2023-09-22T13:10:33.863 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_84 flags=8 2023-09-22T13:10:33.863 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_85 flags=8 2023-09-22T13:10:33.863 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_86 flags=8 2023-09-22T13:10:33.863 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_87 flags=8 2023-09-22T13:10:33.863 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_88 flags=8 2023-09-22T13:10:33.863 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_89 flags=8 2023-09-22T13:10:33.863 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_9 flags=8 2023-09-22T13:10:33.863 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_90 flags=8 2023-09-22T13:10:33.863 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_91 flags=8 2023-09-22T13:10:33.863 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_92 flags=8 2023-09-22T13:10:33.864 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_93 flags=8 2023-09-22T13:10:33.864 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_94 flags=8 2023-09-22T13:10:33.864 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_95 flags=8 2023-09-22T13:10:33.864 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_96 flags=8 2023-09-22T13:10:33.864 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_97 flags=8 2023-09-22T13:10:33.864 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_98 flags=8 2023-09-22T13:10:33.864 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_99 flags=8 2023-09-22T13:10:33.864 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER2_READDIR (2 ms) 2023-09-22T13:10:33.864 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:10:33.864 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:10:33.865 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:10:33.865 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:10:33.865 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:10:33.865 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:10:33.865 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:10:33.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:33.837+0000 7f8defdd3540 -1 shutting down 2023-09-22T13:10:33.865 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:33.837+0000 7f8cf77fe700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156626, si_uid : 1000, si_addr0x3e8000263d2, si_status0 2023-09-22T13:10:33.902 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SHUTDOWN (65 ms) 2023-09-22T13:10:33.902 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 11 tests from LibRGW (795 ms total) 2023-09-22T13:10:33.902 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:33.903 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:33.903 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 11 tests from 1 test suite ran. (795 ms total) 2023-09-22T13:10:33.903 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 11 tests. 2023-09-22T13:10:33.904 INFO:tasks.workunit.client.0.smithi170.stdout:phase 4.2 2023-09-22T13:10:33.920 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 11 tests from 1 test suite. 2023-09-22T13:10:33.920 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:33.920 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 11 tests from LibRGW 2023-09-22T13:10:33.920 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.TVAR 2023-09-22T13:10:33.920 INFO:tasks.workunit.client.0.smithi170.stdout:read i1: 64001 2023-09-22T13:10:33.920 INFO:tasks.workunit.client.0.smithi170.stdout:read s1: blunderbuss 2023-09-22T13:10:33.920 INFO:tasks.workunit.client.0.smithi170.stdout:read s3: [ OK ] LibRGW.TVAR (0 ms) 2023-09-22T13:10:33.920 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:10:33.928 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:33.928 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:34.022 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.INIT (102 ms) 2023-09-22T13:10:34.022 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:10:34.023 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:10:34.023 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:10:34.038 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (15 ms) 2023-09-22T13:10:34.039 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:10:34.039 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:10:34.039 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:10:34.039 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:10:34.039 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER2_READDIR 2023-09-22T13:10:34.042 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_0 flags=8 2023-09-22T13:10:34.042 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_1 flags=8 2023-09-22T13:10:34.042 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_10 flags=8 2023-09-22T13:10:34.042 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_11 flags=8 2023-09-22T13:10:34.042 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_12 flags=8 2023-09-22T13:10:34.042 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_13 flags=8 2023-09-22T13:10:34.042 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_14 flags=8 2023-09-22T13:10:34.042 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_15 flags=8 2023-09-22T13:10:34.042 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_16 flags=8 2023-09-22T13:10:34.042 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_17 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_18 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_19 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_2 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_20 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_21 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_22 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_23 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_24 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_25 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_26 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_27 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_28 flags=8 2023-09-22T13:10:34.043 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_29 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_3 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_30 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_31 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_32 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_33 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_34 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_35 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_36 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_37 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_38 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_39 flags=8 2023-09-22T13:10:34.044 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_4 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_40 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_41 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_42 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_43 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_44 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_45 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_46 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_47 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_48 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_49 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_5 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_50 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_51 flags=8 2023-09-22T13:10:34.045 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_52 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_53 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_54 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_55 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_56 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_57 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_58 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_59 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_6 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_60 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_61 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_62 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_63 flags=8 2023-09-22T13:10:34.046 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_64 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_65 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_66 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_67 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_68 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_69 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_7 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_70 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_71 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_72 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_73 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_74 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_75 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_76 flags=8 2023-09-22T13:10:34.047 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_77 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_78 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_79 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_8 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_80 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_81 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_82 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_83 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_84 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_85 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_86 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_87 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_88 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_89 flags=8 2023-09-22T13:10:34.048 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_9 flags=8 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_90 flags=8 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_91 flags=8 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_92 flags=8 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_93 flags=8 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_94 flags=8 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_95 flags=8 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_96 flags=8 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_97 flags=8 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_98 flags=8 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_99 flags=8 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER2_READDIR (3 ms) 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:10:34.049 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:10:34.050 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:10:34.050 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:10:34.050 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:10:34.050 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:10:34.050 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:34.041+0000 7fe889d51540 -1 shutting down 2023-09-22T13:10:34.123 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SHUTDOWN (82 ms) 2023-09-22T13:10:34.123 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 11 tests from LibRGW (203 ms total) 2023-09-22T13:10:34.123 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:34.123 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:34.123 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 11 tests from 1 test suite ran. (203 ms total) 2023-09-22T13:10:34.123 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 11 tests. 2023-09-22T13:10:34.125 INFO:tasks.workunit.client.0.smithi170.stdout:phase 5.1 2023-09-22T13:10:34.164 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 19 tests from 1 test suite. 2023-09-22T13:10:34.164 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:34.164 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 19 tests from LibRGW 2023-09-22T13:10:34.164 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:10:34.173 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:34.173 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:34.255 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.INIT (92 ms) 2023-09-22T13:10:34.255 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:10:34.256 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:10:34.257 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:10:34.265 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CREATE_BUCKET (9 ms) 2023-09-22T13:10:34.265 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:10:34.269 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (4 ms) 2023-09-22T13:10:34.269 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LIST_OBJECTS 2023-09-22T13:10:34.269 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LIST_OBJECTS (0 ms) 2023-09-22T13:10:34.269 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:10:34.271 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (2 ms) 2023-09-22T13:10:34.271 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.OBJ_OPEN 2023-09-22T13:10:34.271 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.OBJ_OPEN (0 ms) 2023-09-22T13:10:34.271 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.PUT_OBJECT 2023-09-22T13:10:34.275 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.PUT_OBJECT (4 ms) 2023-09-22T13:10:34.276 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.GET_OBJECT 2023-09-22T13:10:34.277 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.GET_OBJECT (2 ms) 2023-09-22T13:10:34.277 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:10:34.277 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:10:34.277 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.WRITE_READ_VERIFY 2023-09-22T13:10:34.278 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.WRITE_READ_VERIFY (0 ms) 2023-09-22T13:10:34.278 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.WRITEV 2023-09-22T13:10:34.278 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.WRITEV (0 ms) 2023-09-22T13:10:34.278 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READV 2023-09-22T13:10:34.278 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READV (0 ms) 2023-09-22T13:10:34.278 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READV_AFTER_WRITEV 2023-09-22T13:10:34.278 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READV_AFTER_WRITEV (0 ms) 2023-09-22T13:10:34.278 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:10:34.278 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_OBJECT (0 ms) 2023-09-22T13:10:34.278 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:10:34.278 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_BUCKET (0 ms) 2023-09-22T13:10:34.279 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:10:34.279 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:10:34.279 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:10:34.279 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:10:34.279 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:10:34.279 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:34.277+0000 7f2ffb561540 -1 shutting down 2023-09-22T13:10:34.279 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:34.277+0000 7f2f02ffd700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156794, si_uid : 1000, si_addr0x3e80002647a, si_status0 2023-09-22T13:10:34.356 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SHUTDOWN (79 ms) 2023-09-22T13:10:34.356 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 19 tests from LibRGW (193 ms total) 2023-09-22T13:10:34.356 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:34.357 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:34.357 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 19 tests from 1 test suite ran. (193 ms total) 2023-09-22T13:10:34.357 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 19 tests. 2023-09-22T13:10:34.358 INFO:tasks.workunit.client.0.smithi170.stdout:phase 5.2 2023-09-22T13:10:34.397 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] Running 19 tests from 1 test suite. 2023-09-22T13:10:34.398 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:34.398 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 19 tests from LibRGW 2023-09-22T13:10:34.398 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:10:34.404 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:34.404 INFO:tasks.workunit.client.0.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:34.503 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.INIT (106 ms) 2023-09-22T13:10:34.503 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:10:34.504 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:10:34.504 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:10:34.504 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CREATE_BUCKET (0 ms) 2023-09-22T13:10:34.504 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:10:34.511 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (7 ms) 2023-09-22T13:10:34.511 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LIST_OBJECTS 2023-09-22T13:10:34.511 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LIST_OBJECTS (0 ms) 2023-09-22T13:10:34.512 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:10:34.512 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (0 ms) 2023-09-22T13:10:34.512 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.OBJ_OPEN 2023-09-22T13:10:34.512 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.OBJ_OPEN (0 ms) 2023-09-22T13:10:34.512 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.PUT_OBJECT 2023-09-22T13:10:34.512 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.PUT_OBJECT (0 ms) 2023-09-22T13:10:34.512 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.GET_OBJECT 2023-09-22T13:10:34.512 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.GET_OBJECT (0 ms) 2023-09-22T13:10:34.512 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:10:34.512 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:10:34.512 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.WRITE_READ_VERIFY 2023-09-22T13:10:34.513 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.WRITE_READ_VERIFY (0 ms) 2023-09-22T13:10:34.513 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.WRITEV 2023-09-22T13:10:34.513 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.WRITEV (0 ms) 2023-09-22T13:10:34.513 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READV 2023-09-22T13:10:34.513 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READV (0 ms) 2023-09-22T13:10:34.513 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.READV_AFTER_WRITEV 2023-09-22T13:10:34.513 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.READV_AFTER_WRITEV (0 ms) 2023-09-22T13:10:34.513 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:10:34.517 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_OBJECT (4 ms) 2023-09-22T13:10:34.517 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:10:34.550 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.DELETE_BUCKET (34 ms) 2023-09-22T13:10:34.550 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:10:34.550 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:10:34.550 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:10:34.550 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:10:34.550 INFO:tasks.workunit.client.0.smithi170.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:10:34.551 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:34.549+0000 7f70be30b540 -1 shutting down 2023-09-22T13:10:34.551 INFO:tasks.workunit.client.0.smithi170.stderr:2023-09-22T13:10:34.549+0000 7f6fd5ffb700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 156878, si_uid : 1000, si_addr0x3e8000264ce, si_status0 2023-09-22T13:10:34.604 INFO:tasks.workunit.client.0.smithi170.stdout:[ OK ] LibRGW.SHUTDOWN (55 ms) 2023-09-22T13:10:34.604 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] 19 tests from LibRGW (207 ms total) 2023-09-22T13:10:34.604 INFO:tasks.workunit.client.0.smithi170.stdout: 2023-09-22T13:10:34.604 INFO:tasks.workunit.client.0.smithi170.stdout:[----------] Global test environment tear-down 2023-09-22T13:10:34.604 INFO:tasks.workunit.client.0.smithi170.stdout:[==========] 19 tests from 1 test suite ran. (207 ms total) 2023-09-22T13:10:34.604 INFO:tasks.workunit.client.0.smithi170.stdout:[ PASSED ] 19 tests. 2023-09-22T13:10:34.607 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:10:34.607 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:10:34.635 INFO:tasks.workunit:Stopping ['cls/test_cls_lock.sh', 'cls/test_cls_log.sh', 'cls/test_cls_refcount.sh', 'cls/test_cls_rgw.sh', 'cls/test_cls_cmpomap.sh', 'cls/test_cls_2pc_queue.sh', 'rgw/test_rgw_gc_log.sh', 'rgw/test_rgw_obj.sh', 'rgw/test_rgw_throttle.sh', 'rgw/test_librgw_file.sh'] on client.0... 2023-09-22T13:10:34.636 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-22T13:10:34.932 DEBUG:teuthology.parallel:result is None 2023-09-22T13:10:34.933 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:10:34.960 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:10:34.961 DEBUG:teuthology.orchestra.run.smithi170:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:10:35.018 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:10:35.018 INFO:teuthology.run_tasks:Running task ragweed... 2023-09-22T13:10:35.031 DEBUG:tasks.ragweed:ragweed config is {'client.0': {'default-branch': 'ceph-pacific', 'rgw_server': 'client.0', 'stages': 'prepare,check'}} 2023-09-22T13:10:35.031 INFO:tasks.ragweed:Downloading ragweed... 2023-09-22T13:10:35.032 INFO:tasks.ragweed:Using branch 'wip-cfsnyder-bucket-check-pacific' for ragweed 2023-09-22T13:10:35.032 DEBUG:teuthology.orchestra.run.smithi170:> git clone -b wip-cfsnyder-bucket-check-pacific https://git.ceph.com/ragweed.git /home/ubuntu/cephtest/ragweed 2023-09-22T13:10:35.076 INFO:teuthology.orchestra.run.smithi170.stderr:Cloning into '/home/ubuntu/cephtest/ragweed'... 2023-09-22T13:10:35.153 INFO:teuthology.orchestra.run.smithi170.stderr:fatal: Remote branch wip-cfsnyder-bucket-check-pacific not found in upstream origin 2023-09-22T13:10:35.154 DEBUG:teuthology.orchestra.run:got remote process result: 128 2023-09-22T13:10:35.155 INFO:tasks.ragweed:Using branch 'ceph-pacific' for ragweed 2023-09-22T13:10:35.155 DEBUG:teuthology.orchestra.run.smithi170:> git clone -b ceph-pacific https://git.ceph.com/ragweed.git /home/ubuntu/cephtest/ragweed 2023-09-22T13:10:35.214 INFO:teuthology.orchestra.run.smithi170.stderr:Cloning into '/home/ubuntu/cephtest/ragweed'... 2023-09-22T13:10:35.406 INFO:tasks.ragweed:Creating rgw users... 2023-09-22T13:10:35.406 DEBUG:tasks.ragweed:Creating user ragweed.client.0 on client.0 2023-09-22T13:10:35.407 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user create --uid ragweed.client.0 --display-name 'Mr. ragweed.client.0' --access-key WFZCZRDVLGCNMTJJRKXK --secret vMN9c4Pz+IEJeeJNmEhLSEE8rqdCX+wGyhc8/v/ZbGwuzOuQjAx+1g== --email ragweed.client.0+test@test.test --system false 2023-09-22T13:10:35.495 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:35.496 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:35.501 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.501+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.503 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.503+0000 7efda2785500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.503 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.503+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.504 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.503+0000 7efda2785500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.504 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.503+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.504 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.503+0000 7efda2785500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.505 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.503+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.505 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.504+0000 7efda2785500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:35.516 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.515+0000 7efda2785500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:10:35.516 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.515+0000 7efda2785500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:10:35.516 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.515+0000 7efda2785500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:10:35.516 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.515+0000 7efda2785500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:10:35.517 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.515+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.517 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.515+0000 7efda2785500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:35.517 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.515+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.517 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.515+0000 7efda2785500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:10:35.517 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.515+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.517 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.515+0000 7efda2785500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:35.517 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.515+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.520 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.516+0000 7efda2785500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:10:35.520 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.516+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.520 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.516+0000 7efda2785500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.520 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.516+0000 7efda2785500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:10:35.520 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.516+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.521 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.516+0000 7efda2785500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.521 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.516+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.521 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.517+0000 7efda2785500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:35.521 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.517+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.521 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.517+0000 7efda2785500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:10:35.521 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.517+0000 7efda2785500 20 zonegroup default 2023-09-22T13:10:35.521 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.517+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.521 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.517+0000 7efda2785500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.522 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.517+0000 7efda2785500 10 Cannot find current period zone using local zone 2023-09-22T13:10:35.522 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.517+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.522 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.518+0000 7efda2785500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.522 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.518+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.522 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.518+0000 7efda2785500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:35.522 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.518+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.522 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.518+0000 7efda2785500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:10:35.522 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.518+0000 7efda2785500 20 zone default found 2023-09-22T13:10:35.523 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.518+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.523 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.518+0000 7efda2785500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.523 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.518+0000 7efda2785500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.523 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.519+0000 7efda2785500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.523 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.519+0000 7efda2785500 20 started sync module instance, tier type = 2023-09-22T13:10:35.523 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.519+0000 7efda2785500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:10:35.528 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.528+0000 7efda2785500 20 add_watcher() i=0 2023-09-22T13:10:35.529 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.528+0000 7efda2785500 20 add_watcher() i=1 2023-09-22T13:10:35.529 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.528+0000 7efda2785500 20 add_watcher() i=2 2023-09-22T13:10:35.529 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.528+0000 7efda2785500 20 add_watcher() i=3 2023-09-22T13:10:35.529 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.528+0000 7efda2785500 20 add_watcher() i=4 2023-09-22T13:10:35.529 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.528+0000 7efda2785500 20 add_watcher() i=5 2023-09-22T13:10:35.529 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.528+0000 7efda2785500 20 add_watcher() i=6 2023-09-22T13:10:35.530 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.529+0000 7efda2785500 20 add_watcher() i=7 2023-09-22T13:10:35.530 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.529+0000 7efda2785500 2 all 8 watchers are set, enabling cache 2023-09-22T13:10:35.530 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.529+0000 7efd8e468700 0 -- 172.21.15.170:0/2691605114 >> [v2:172.21.15.191:6816/144992,v1:172.21.15.191:6817/144992] conn(0x7efd84022930 msgr2=0x7efd84014390 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:10:35.532 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.531+0000 7efd5efe5700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:10:35.532 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.531+0000 7efda2785500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:10:35.532 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.531+0000 7efda2785500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:10:35.535 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.534+0000 7efd8e468700 0 -- 172.21.15.170:0/2992688275 >> [v2:172.21.15.170:3300/0,v1:172.21.15.170:6789/0] conn(0x5600ae857410 msgr2=0x5600ae85da00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:10:35.537 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.536+0000 7efda2785500 5 note: GC not initialized 2023-09-22T13:10:35.539 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.537+0000 7efccdffb700 20 reqs_thread_entry: start 2023-09-22T13:10:35.541 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.537+0000 7efcbf7fe700 20 reqs_thread_entry: start 2023-09-22T13:10:35.541 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.537+0000 7efda2785500 20 init_complete bucket index max shards: 11 2023-09-22T13:10:35.541 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.537+0000 7efda2785500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:10:35.541 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.537+0000 7efcbdffb700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:11:05 2023 2023-09-22T13:10:35.541 INFO:teuthology.orchestra.run.smithi170.stderr: 2023-09-22T13:10:35.541 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.537+0000 7efcbd7fa700 20 reqs_thread_entry: start 2023-09-22T13:10:35.541 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.537+0000 7efda2785500 20 get_system_obj_state: rctx=0x7ffc4a516af8 obj=default.rgw.meta:users.uid:ragweed.client.0 state=0x5600ae6f6810 s->prefetch_data=0 2023-09-22T13:10:35.542 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.537+0000 7efda2785500 10 cache get: name=default.rgw.meta+users.uid+ragweed.client.0 : miss 2023-09-22T13:10:35.542 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.538+0000 7efda2785500 10 cache put: name=default.rgw.meta+users.uid+ragweed.client.0 info.flags=0x0 2023-09-22T13:10:35.542 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.538+0000 7efda2785500 10 adding default.rgw.meta+users.uid+ragweed.client.0 to cache LRU end 2023-09-22T13:10:35.542 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.538+0000 7efda2785500 20 get_system_obj_state: rctx=0x7ffc4a516b08 obj=default.rgw.meta:users.email:ragweed.client.0+test@test.test state=0x5600ae6f6810 s->prefetch_data=0 2023-09-22T13:10:35.542 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.538+0000 7efda2785500 10 cache get: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test : miss 2023-09-22T13:10:35.542 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.538+0000 7efda2785500 10 cache put: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test info.flags=0x0 2023-09-22T13:10:35.542 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.538+0000 7efda2785500 10 adding default.rgw.meta+users.email+ragweed.client.0+test@test.test to cache LRU end 2023-09-22T13:10:35.542 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.538+0000 7efda2785500 20 get_system_obj_state: rctx=0x7ffc4a516b08 obj=default.rgw.meta:users.keys:WFZCZRDVLGCNMTJJRKXK state=0x5600ae6f6810 s->prefetch_data=0 2023-09-22T13:10:35.543 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.538+0000 7efda2785500 10 cache get: name=default.rgw.meta+users.keys+WFZCZRDVLGCNMTJJRKXK : miss 2023-09-22T13:10:35.543 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.539+0000 7efda2785500 10 cache put: name=default.rgw.meta+users.keys+WFZCZRDVLGCNMTJJRKXK info.flags=0x0 2023-09-22T13:10:35.543 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.539+0000 7efda2785500 10 adding default.rgw.meta+users.keys+WFZCZRDVLGCNMTJJRKXK to cache LRU end 2023-09-22T13:10:35.543 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.539+0000 7efda2785500 20 get_system_obj_state: rctx=0x7ffc4a516408 obj=default.rgw.meta:users.keys:WFZCZRDVLGCNMTJJRKXK state=0x5600ae6f6810 s->prefetch_data=0 2023-09-22T13:10:35.543 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.539+0000 7efda2785500 10 cache get: name=default.rgw.meta+users.keys+WFZCZRDVLGCNMTJJRKXK : hit (negative entry) 2023-09-22T13:10:35.543 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.539+0000 7efda2785500 20 get_system_obj_state: rctx=0x7ffc4a516748 obj=default.rgw.meta:users.keys:WFZCZRDVLGCNMTJJRKXK state=0x5600ae6f6810 s->prefetch_data=0 2023-09-22T13:10:35.543 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.539+0000 7efda2785500 10 cache get: name=default.rgw.meta+users.keys+WFZCZRDVLGCNMTJJRKXK : hit (negative entry) 2023-09-22T13:10:35.545 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.540+0000 7efda2785500 10 cache put: name=default.rgw.meta+users.uid+ragweed.client.0 info.flags=0x17 2023-09-22T13:10:35.545 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.540+0000 7efda2785500 10 moving default.rgw.meta+users.uid+ragweed.client.0 to cache LRU end 2023-09-22T13:10:35.545 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.540+0000 7efda2785500 10 distributing notification oid=default.rgw.control:notify.4 cni=[op: 0, obj: default.rgw.meta:users.uid:ragweed.client.0, ofs0, ns] 2023-09-22T13:10:35.545 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.540+0000 7efd7f7fe700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1017907249285 cookie 94560927905072 notifier 5357 bl.length()=641 2023-09-22T13:10:35.545 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.540+0000 7efd7f7fe700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+ragweed.client.0 info.flags=0x17 2023-09-22T13:10:35.546 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.540+0000 7efd7f7fe700 10 rgw watcher librados: moving default.rgw.meta+users.uid+ragweed.client.0 to cache LRU end 2023-09-22T13:10:35.546 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.542+0000 7efda2785500 10 cache put: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test info.flags=0x7 2023-09-22T13:10:35.546 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.542+0000 7efda2785500 10 moving default.rgw.meta+users.email+ragweed.client.0+test@test.test to cache LRU end 2023-09-22T13:10:35.546 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.542+0000 7efda2785500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.email:ragweed.client.0+test@test.test, ofs0, ns] 2023-09-22T13:10:35.546 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.542+0000 7efd7effd700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1017907249165 cookie 94560927898160 notifier 5357 bl.length()=202 2023-09-22T13:10:35.546 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.542+0000 7efd7effd700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test info.flags=0x7 2023-09-22T13:10:35.546 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.542+0000 7efd7effd700 10 rgw watcher librados: moving default.rgw.meta+users.email+ragweed.client.0+test@test.test to cache LRU end 2023-09-22T13:10:35.546 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.544+0000 7efda2785500 10 cache put: name=default.rgw.meta+users.keys+WFZCZRDVLGCNMTJJRKXK info.flags=0x7 2023-09-22T13:10:35.547 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.544+0000 7efda2785500 10 moving default.rgw.meta+users.keys+WFZCZRDVLGCNMTJJRKXK to cache LRU end 2023-09-22T13:10:35.547 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.544+0000 7efda2785500 10 distributing notification oid=default.rgw.control:notify.3 cni=[op: 0, obj: default.rgw.meta:users.keys:WFZCZRDVLGCNMTJJRKXK, ofs0, ns] 2023-09-22T13:10:35.547 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.544+0000 7efd7f7fe700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1017907249164 cookie 94560927899552 notifier 5357 bl.length()=190 2023-09-22T13:10:35.547 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.544+0000 7efd7f7fe700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+WFZCZRDVLGCNMTJJRKXK info.flags=0x7 2023-09-22T13:10:35.547 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.544+0000 7efd7f7fe700 10 rgw watcher librados: moving default.rgw.meta+users.keys+WFZCZRDVLGCNMTJJRKXK to cache LRU end 2023-09-22T13:10:35.548 INFO:teuthology.orchestra.run.smithi170.stdout:{ 2023-09-22T13:10:35.548 INFO:teuthology.orchestra.run.smithi170.stdout: "user_id": "ragweed.client.0", 2023-09-22T13:10:35.548 INFO:teuthology.orchestra.run.smithi170.stdout: "display_name": "Mr. ragweed.client.0", 2023-09-22T13:10:35.548 INFO:teuthology.orchestra.run.smithi170.stdout: "email": "ragweed.client.0+test@test.test", 2023-09-22T13:10:35.548 INFO:teuthology.orchestra.run.smithi170.stdout: "suspended": 0, 2023-09-22T13:10:35.548 INFO:teuthology.orchestra.run.smithi170.stdout: "max_buckets": 1000, 2023-09-22T13:10:35.548 INFO:teuthology.orchestra.run.smithi170.stdout: "subusers": [], 2023-09-22T13:10:35.548 INFO:teuthology.orchestra.run.smithi170.stdout: "keys": [ 2023-09-22T13:10:35.549 INFO:teuthology.orchestra.run.smithi170.stdout: { 2023-09-22T13:10:35.549 INFO:teuthology.orchestra.run.smithi170.stdout: "user": "ragweed.client.0", 2023-09-22T13:10:35.549 INFO:teuthology.orchestra.run.smithi170.stdout: "access_key": "WFZCZRDVLGCNMTJJRKXK", 2023-09-22T13:10:35.549 INFO:teuthology.orchestra.run.smithi170.stdout: "secret_key": "vMN9c4Pz+IEJeeJNmEhLSEE8rqdCX+wGyhc8/v/ZbGwuzOuQjAx+1g==" 2023-09-22T13:10:35.549 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:10:35.549 INFO:teuthology.orchestra.run.smithi170.stdout: ], 2023-09-22T13:10:35.549 INFO:teuthology.orchestra.run.smithi170.stdout: "swift_keys": [], 2023-09-22T13:10:35.549 INFO:teuthology.orchestra.run.smithi170.stdout: "caps": [], 2023-09-22T13:10:35.549 INFO:teuthology.orchestra.run.smithi170.stdout: "op_mask": "read, write, delete", 2023-09-22T13:10:35.550 INFO:teuthology.orchestra.run.smithi170.stdout: "default_placement": "", 2023-09-22T13:10:35.550 INFO:teuthology.orchestra.run.smithi170.stdout: "default_storage_class": "", 2023-09-22T13:10:35.550 INFO:teuthology.orchestra.run.smithi170.stdout: "placement_tags": [], 2023-09-22T13:10:35.550 INFO:teuthology.orchestra.run.smithi170.stdout: "bucket_quota": { 2023-09-22T13:10:35.550 INFO:teuthology.orchestra.run.smithi170.stdout: "enabled": false, 2023-09-22T13:10:35.550 INFO:teuthology.orchestra.run.smithi170.stdout: "check_on_raw": false, 2023-09-22T13:10:35.550 INFO:teuthology.orchestra.run.smithi170.stdout: "max_size": -1, 2023-09-22T13:10:35.550 INFO:teuthology.orchestra.run.smithi170.stdout: "max_size_kb": 0, 2023-09-22T13:10:35.551 INFO:teuthology.orchestra.run.smithi170.stdout: "max_objects": -1 2023-09-22T13:10:35.551 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:10:35.551 INFO:teuthology.orchestra.run.smithi170.stdout: "user_quota": { 2023-09-22T13:10:35.551 INFO:teuthology.orchestra.run.smithi170.stdout: "enabled": false, 2023-09-22T13:10:35.551 INFO:teuthology.orchestra.run.smithi170.stdout: "check_on_raw": false, 2023-09-22T13:10:35.551 INFO:teuthology.orchestra.run.smithi170.stdout: "max_size": -1, 2023-09-22T13:10:35.551 INFO:teuthology.orchestra.run.smithi170.stdout: "max_size_kb": 0, 2023-09-22T13:10:35.551 INFO:teuthology.orchestra.run.smithi170.stdout: "max_objects": -1 2023-09-22T13:10:35.551 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:10:35.552 INFO:teuthology.orchestra.run.smithi170.stdout: "temp_url_keys": [], 2023-09-22T13:10:35.552 INFO:teuthology.orchestra.run.smithi170.stdout: "type": "rgw", 2023-09-22T13:10:35.552 INFO:teuthology.orchestra.run.smithi170.stdout: "mfa_ids": [] 2023-09-22T13:10:35.552 INFO:teuthology.orchestra.run.smithi170.stdout:} 2023-09-22T13:10:35.552 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:10:35.553 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.546+0000 7efda2785500 20 remove_watcher() i=0 2023-09-22T13:10:35.553 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.546+0000 7efda2785500 2 removed watcher, disabling cache 2023-09-22T13:10:35.553 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.547+0000 7efda2785500 20 remove_watcher() i=1 2023-09-22T13:10:35.553 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.548+0000 7efda2785500 20 remove_watcher() i=2 2023-09-22T13:10:35.554 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.549+0000 7efda2785500 20 remove_watcher() i=3 2023-09-22T13:10:35.554 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.550+0000 7efda2785500 20 remove_watcher() i=4 2023-09-22T13:10:35.554 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.551+0000 7efda2785500 20 remove_watcher() i=5 2023-09-22T13:10:35.554 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.552+0000 7efda2785500 20 remove_watcher() i=6 2023-09-22T13:10:35.555 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.553+0000 7efda2785500 20 remove_watcher() i=7 2023-09-22T13:10:35.555 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.554+0000 7efd5efe5700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:10:35.559 DEBUG:tasks.ragweed:Creating user sysuser.client.0 on client.0 2023-09-22T13:10:35.560 DEBUG:teuthology.orchestra.run.smithi170:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user create --uid sysuser.client.0 --display-name 'Mr. sysuser.client.0' --access-key QMCABLXIJQRCJAEYJATB --secret 5aNoOneUFXnoHU8M4T6Lf24/NJkJ1Pc+U9xgWRFYVGTsER61Jjh5dg== --email sysuser.client.0+test@test.test --system true 2023-09-22T13:10:35.610 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:10:35.610 INFO:teuthology.orchestra.run.smithi170.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:10:35.617 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.616+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.619 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.618+0000 7f32c9242500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.619 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.618+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.619 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.619+0000 7f32c9242500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.620 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.619+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.620 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.619+0000 7f32c9242500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.620 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.619+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.620 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.619+0000 7f32c9242500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:35.634 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.630+0000 7f32c9242500 20 RGWRados::pool_iterate: got zone_info.772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 2023-09-22T13:10:35.635 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.630+0000 7f32c9242500 20 RGWRados::pool_iterate: got zonegroup_info.060bb046-2656-46be-891f-3e4dd76a5093 2023-09-22T13:10:35.635 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.630+0000 7f32c9242500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:10:35.635 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.630+0000 7f32c9242500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:10:35.635 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.630+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.635 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.630+0000 7f32c9242500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:35.635 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.630+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.635 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.630+0000 7f32c9242500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:10:35.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.630+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.631+0000 7f32c9242500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:35.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.631+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.631+0000 7f32c9242500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:10:35.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.631+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.631+0000 7f32c9242500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.631+0000 7f32c9242500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:10:35.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.631+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.636 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.631+0000 7f32c9242500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.637 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.631+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.637 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.632+0000 7f32c9242500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:35.637 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.632+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.637 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.632+0000 7f32c9242500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:10:35.637 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.632+0000 7f32c9242500 20 zonegroup default 2023-09-22T13:10:35.637 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.632+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.637 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.632+0000 7f32c9242500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.637 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.632+0000 7f32c9242500 10 Cannot find current period zone using local zone 2023-09-22T13:10:35.638 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.632+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.638 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.632+0000 7f32c9242500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.638 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.632+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.638 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.633+0000 7f32c9242500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:10:35.638 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.633+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.638 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.633+0000 7f32c9242500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:10:35.639 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.633+0000 7f32c9242500 20 zone default found 2023-09-22T13:10:35.639 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.633+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.639 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.633+0000 7f32c9242500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.639 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.633+0000 7f32c9242500 20 rados->read ofs=0 len=0 2023-09-22T13:10:35.639 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.633+0000 7f32c9242500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:10:35.639 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.633+0000 7f32c9242500 20 started sync module instance, tier type = 2023-09-22T13:10:35.639 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.633+0000 7f32c9242500 20 started zone id=772ce3cc-30ad-4ea4-82fa-9614e9a1eee1 (name=default) with tier type = 2023-09-22T13:10:35.640 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.637+0000 7f32b4724700 0 -- 172.21.15.170:0/2101142844 >> [v2:172.21.15.170:6824/145036,v1:172.21.15.170:6825/145036] conn(0x7f329c003a70 msgr2=0x7f32ac024910 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:10:35.643 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.642+0000 7f32c9242500 20 add_watcher() i=0 2023-09-22T13:10:35.643 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.642+0000 7f32c9242500 20 add_watcher() i=1 2023-09-22T13:10:35.643 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.642+0000 7f32c9242500 20 add_watcher() i=2 2023-09-22T13:10:35.643 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.642+0000 7f32c9242500 20 add_watcher() i=3 2023-09-22T13:10:35.643 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.642+0000 7f32c9242500 20 add_watcher() i=4 2023-09-22T13:10:35.644 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.642+0000 7f32c9242500 20 add_watcher() i=5 2023-09-22T13:10:35.644 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.642+0000 7f32c9242500 20 add_watcher() i=6 2023-09-22T13:10:35.645 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.644+0000 7f32c9242500 20 add_watcher() i=7 2023-09-22T13:10:35.645 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.644+0000 7f32c9242500 2 all 8 watchers are set, enabling cache 2023-09-22T13:10:35.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.646+0000 7f3285fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:10:35.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.647+0000 7f32c9242500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:10:35.647 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.647+0000 7f32c9242500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:10:35.648 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.647+0000 7f32b3f23700 0 -- 172.21.15.170:0/133906811 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x564774091030 msgr2=0x5647740914b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=0)._try_send injecting socket failure 2023-09-22T13:10:35.849 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.848+0000 7f32b3f23700 0 --2- 172.21.15.170:0/133906811 >> [v2:172.21.15.170:3301/0,v1:172.21.15.170:6790/0] conn(0x564774091030 0x5647740914b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:10:35.853 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.852+0000 7f32c9242500 5 note: GC not initialized 2023-09-22T13:10:35.855 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.853+0000 7f31fcff9700 20 reqs_thread_entry: start 2023-09-22T13:10:35.855 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.854+0000 7f31f27fc700 20 reqs_thread_entry: start 2023-09-22T13:10:35.855 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.854+0000 7f32c9242500 20 init_complete bucket index max shards: 11 2023-09-22T13:10:35.855 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.854+0000 7f32c9242500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:10:35.856 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.854+0000 7f31f0ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:11:06 2023 2023-09-22T13:10:35.856 INFO:teuthology.orchestra.run.smithi170.stderr: 2023-09-22T13:10:35.856 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.854+0000 7f31d7fff700 20 reqs_thread_entry: start 2023-09-22T13:10:35.856 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.854+0000 7f32c9242500 20 get_system_obj_state: rctx=0x7ffeafd431a8 obj=default.rgw.meta:users.uid:sysuser.client.0 state=0x564773f30810 s->prefetch_data=0 2023-09-22T13:10:35.856 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.854+0000 7f32c9242500 10 cache get: name=default.rgw.meta+users.uid+sysuser.client.0 : miss 2023-09-22T13:10:35.856 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 10 cache put: name=default.rgw.meta+users.uid+sysuser.client.0 info.flags=0x0 2023-09-22T13:10:35.856 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 10 adding default.rgw.meta+users.uid+sysuser.client.0 to cache LRU end 2023-09-22T13:10:35.856 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 20 get_system_obj_state: rctx=0x7ffeafd431b8 obj=default.rgw.meta:users.email:sysuser.client.0+test@test.test state=0x564773f30810 s->prefetch_data=0 2023-09-22T13:10:35.857 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 10 cache get: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test : miss 2023-09-22T13:10:35.858 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 10 cache put: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test info.flags=0x0 2023-09-22T13:10:35.858 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 10 adding default.rgw.meta+users.email+sysuser.client.0+test@test.test to cache LRU end 2023-09-22T13:10:35.858 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 20 get_system_obj_state: rctx=0x7ffeafd431b8 obj=default.rgw.meta:users.keys:QMCABLXIJQRCJAEYJATB state=0x564773f30810 s->prefetch_data=0 2023-09-22T13:10:35.858 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 10 cache get: name=default.rgw.meta+users.keys+QMCABLXIJQRCJAEYJATB : miss 2023-09-22T13:10:35.858 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 10 cache put: name=default.rgw.meta+users.keys+QMCABLXIJQRCJAEYJATB info.flags=0x0 2023-09-22T13:10:35.859 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 10 adding default.rgw.meta+users.keys+QMCABLXIJQRCJAEYJATB to cache LRU end 2023-09-22T13:10:35.859 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 20 get_system_obj_state: rctx=0x7ffeafd42ab8 obj=default.rgw.meta:users.keys:QMCABLXIJQRCJAEYJATB state=0x564773f30810 s->prefetch_data=0 2023-09-22T13:10:35.859 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 10 cache get: name=default.rgw.meta+users.keys+QMCABLXIJQRCJAEYJATB : hit (negative entry) 2023-09-22T13:10:35.859 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 20 get_system_obj_state: rctx=0x7ffeafd42df8 obj=default.rgw.meta:users.keys:QMCABLXIJQRCJAEYJATB state=0x564773f30810 s->prefetch_data=0 2023-09-22T13:10:35.859 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.855+0000 7f32c9242500 10 cache get: name=default.rgw.meta+users.keys+QMCABLXIJQRCJAEYJATB : hit (negative entry) 2023-09-22T13:10:35.859 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.857+0000 7f32c9242500 10 cache put: name=default.rgw.meta+users.uid+sysuser.client.0 info.flags=0x17 2023-09-22T13:10:35.859 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.857+0000 7f32c9242500 10 moving default.rgw.meta+users.uid+sysuser.client.0 to cache LRU end 2023-09-22T13:10:35.859 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.857+0000 7f32c9242500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.uid:sysuser.client.0, ofs0, ns] 2023-09-22T13:10:35.860 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.857+0000 7f32b2720700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1017907249166 cookie 94864889335936 notifier 5682 bl.length()=641 2023-09-22T13:10:35.860 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.857+0000 7f32b2720700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+sysuser.client.0 info.flags=0x17 2023-09-22T13:10:35.860 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.857+0000 7f32b2720700 10 rgw watcher librados: moving default.rgw.meta+users.uid+sysuser.client.0 to cache LRU end 2023-09-22T13:10:35.861 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.859+0000 7f32c9242500 10 cache put: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test info.flags=0x7 2023-09-22T13:10:35.861 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.859+0000 7f32c9242500 10 moving default.rgw.meta+users.email+sysuser.client.0+test@test.test to cache LRU end 2023-09-22T13:10:35.861 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.859+0000 7f32c9242500 10 distributing notification oid=default.rgw.control:notify.4 cni=[op: 0, obj: default.rgw.meta:users.email:sysuser.client.0+test@test.test, ofs0, ns] 2023-09-22T13:10:35.861 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.860+0000 7f32b2720700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1017907249286 cookie 94864889348384 notifier 5682 bl.length()=202 2023-09-22T13:10:35.861 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.860+0000 7f32b2720700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test info.flags=0x7 2023-09-22T13:10:35.861 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.860+0000 7f32b2720700 10 rgw watcher librados: moving default.rgw.meta+users.email+sysuser.client.0+test@test.test to cache LRU end 2023-09-22T13:10:35.862 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.861+0000 7f32c9242500 10 cache put: name=default.rgw.meta+users.keys+QMCABLXIJQRCJAEYJATB info.flags=0x7 2023-09-22T13:10:35.862 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.861+0000 7f32c9242500 10 moving default.rgw.meta+users.keys+QMCABLXIJQRCJAEYJATB to cache LRU end 2023-09-22T13:10:35.863 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.861+0000 7f32c9242500 10 distributing notification oid=default.rgw.control:notify.7 cni=[op: 0, obj: default.rgw.meta:users.keys:QMCABLXIJQRCJAEYJATB, ofs0, ns] 2023-09-22T13:10:35.863 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.862+0000 7f32b2720700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 1017907249164 cookie 94864889356912 notifier 5682 bl.length()=190 2023-09-22T13:10:35.863 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.862+0000 7f32b2720700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+QMCABLXIJQRCJAEYJATB info.flags=0x7 2023-09-22T13:10:35.863 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.862+0000 7f32b2720700 10 rgw watcher librados: moving default.rgw.meta+users.keys+QMCABLXIJQRCJAEYJATB to cache LRU end 2023-09-22T13:10:35.864 INFO:teuthology.orchestra.run.smithi170.stdout:{ 2023-09-22T13:10:35.864 INFO:teuthology.orchestra.run.smithi170.stdout: "user_id": "sysuser.client.0", 2023-09-22T13:10:35.864 INFO:teuthology.orchestra.run.smithi170.stdout: "display_name": "Mr. sysuser.client.0", 2023-09-22T13:10:35.864 INFO:teuthology.orchestra.run.smithi170.stdout: "email": "sysuser.client.0+test@test.test", 2023-09-22T13:10:35.864 INFO:teuthology.orchestra.run.smithi170.stdout: "suspended": 0, 2023-09-22T13:10:35.864 INFO:teuthology.orchestra.run.smithi170.stdout: "max_buckets": 1000, 2023-09-22T13:10:35.864 INFO:teuthology.orchestra.run.smithi170.stdout: "subusers": [], 2023-09-22T13:10:35.865 INFO:teuthology.orchestra.run.smithi170.stdout: "keys": [ 2023-09-22T13:10:35.865 INFO:teuthology.orchestra.run.smithi170.stdout: { 2023-09-22T13:10:35.865 INFO:teuthology.orchestra.run.smithi170.stdout: "user": "sysuser.client.0", 2023-09-22T13:10:35.865 INFO:teuthology.orchestra.run.smithi170.stdout: "access_key": "QMCABLXIJQRCJAEYJATB", 2023-09-22T13:10:35.865 INFO:teuthology.orchestra.run.smithi170.stdout: "secret_key": "5aNoOneUFXnoHU8M4T6Lf24/NJkJ1Pc+U9xgWRFYVGTsER61Jjh5dg==" 2023-09-22T13:10:35.865 INFO:teuthology.orchestra.run.smithi170.stdout: } 2023-09-22T13:10:35.865 INFO:teuthology.orchestra.run.smithi170.stdout: ], 2023-09-22T13:10:35.865 INFO:teuthology.orchestra.run.smithi170.stdout: "swift_keys": [], 2023-09-22T13:10:35.866 INFO:teuthology.orchestra.run.smithi170.stdout: "caps": [], 2023-09-22T13:10:35.866 INFO:teuthology.orchestra.run.smithi170.stdout: "op_mask": "read, write, delete", 2023-09-22T13:10:35.866 INFO:teuthology.orchestra.run.smithi170.stdout: "system": "true", 2023-09-22T13:10:35.866 INFO:teuthology.orchestra.run.smithi170.stdout: "default_placement": "", 2023-09-22T13:10:35.866 INFO:teuthology.orchestra.run.smithi170.stdout: "default_storage_class": "", 2023-09-22T13:10:35.866 INFO:teuthology.orchestra.run.smithi170.stdout: "placement_tags": [], 2023-09-22T13:10:35.866 INFO:teuthology.orchestra.run.smithi170.stdout: "bucket_quota": { 2023-09-22T13:10:35.866 INFO:teuthology.orchestra.run.smithi170.stdout: "enabled": false, 2023-09-22T13:10:35.866 INFO:teuthology.orchestra.run.smithi170.stdout: "check_on_raw": false, 2023-09-22T13:10:35.867 INFO:teuthology.orchestra.run.smithi170.stdout: "max_size": -1, 2023-09-22T13:10:35.867 INFO:teuthology.orchestra.run.smithi170.stdout: "max_size_kb": 0, 2023-09-22T13:10:35.867 INFO:teuthology.orchestra.run.smithi170.stdout: "max_objects": -1 2023-09-22T13:10:35.867 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:10:35.867 INFO:teuthology.orchestra.run.smithi170.stdout: "user_quota": { 2023-09-22T13:10:35.867 INFO:teuthology.orchestra.run.smithi170.stdout: "enabled": false, 2023-09-22T13:10:35.867 INFO:teuthology.orchestra.run.smithi170.stdout: "check_on_raw": false, 2023-09-22T13:10:35.867 INFO:teuthology.orchestra.run.smithi170.stdout: "max_size": -1, 2023-09-22T13:10:35.867 INFO:teuthology.orchestra.run.smithi170.stdout: "max_size_kb": 0, 2023-09-22T13:10:35.868 INFO:teuthology.orchestra.run.smithi170.stdout: "max_objects": -1 2023-09-22T13:10:35.868 INFO:teuthology.orchestra.run.smithi170.stdout: }, 2023-09-22T13:10:35.868 INFO:teuthology.orchestra.run.smithi170.stdout: "temp_url_keys": [], 2023-09-22T13:10:35.868 INFO:teuthology.orchestra.run.smithi170.stdout: "type": "rgw", 2023-09-22T13:10:35.868 INFO:teuthology.orchestra.run.smithi170.stdout: "mfa_ids": [] 2023-09-22T13:10:35.868 INFO:teuthology.orchestra.run.smithi170.stdout:} 2023-09-22T13:10:35.868 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:10:35.869 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.863+0000 7f32c9242500 20 remove_watcher() i=0 2023-09-22T13:10:35.869 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.863+0000 7f32c9242500 2 removed watcher, disabling cache 2023-09-22T13:10:35.870 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.865+0000 7f32c9242500 20 remove_watcher() i=1 2023-09-22T13:10:35.870 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.866+0000 7f32c9242500 20 remove_watcher() i=2 2023-09-22T13:10:35.870 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.867+0000 7f32c9242500 20 remove_watcher() i=3 2023-09-22T13:10:35.870 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.868+0000 7f32c9242500 20 remove_watcher() i=4 2023-09-22T13:10:35.870 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.869+0000 7f32c9242500 20 remove_watcher() i=5 2023-09-22T13:10:35.871 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.870+0000 7f32c9242500 20 remove_watcher() i=6 2023-09-22T13:10:35.871 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.871+0000 7f32c9242500 20 remove_watcher() i=7 2023-09-22T13:10:35.872 INFO:teuthology.orchestra.run.smithi170.stderr:2023-09-22T13:10:35.871+0000 7f3285fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:10:35.877 INFO:tasks.ragweed:Configuring ragweed... 2023-09-22T13:10:35.877 DEBUG:teuthology.orchestra.run.smithi170:> cd /home/ubuntu/cephtest/ragweed && ./bootstrap 2023-09-22T13:10:35.895 INFO:teuthology.orchestra.run.smithi170.stderr:+ set -e 2023-09-22T13:10:35.896 INFO:teuthology.orchestra.run.smithi170.stderr:+ '[' -f /etc/debian_version ']' 2023-09-22T13:10:35.896 INFO:teuthology.orchestra.run.smithi170.stderr:+ '[' -f /etc/redhat-release ']' 2023-09-22T13:10:35.896 INFO:teuthology.orchestra.run.smithi170.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:10:35.896 INFO:teuthology.orchestra.run.smithi170.stderr:++ rpm -qa python3-pip 2023-09-22T13:10:37.038 INFO:teuthology.orchestra.run.smithi170.stderr:+ '[' python3-pip-9.0.3-22.el8.noarch == '' ']' 2023-09-22T13:10:37.038 INFO:teuthology.orchestra.run.smithi170.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:10:37.039 INFO:teuthology.orchestra.run.smithi170.stderr:++ rpm -qa python3-virtualenv 2023-09-22T13:10:38.176 INFO:teuthology.orchestra.run.smithi170.stderr:+ '[' python3-virtualenv-15.1.0-21.module_el8.5.0+895+a459eca8.noarch == '' ']' 2023-09-22T13:10:38.176 INFO:teuthology.orchestra.run.smithi170.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:10:38.177 INFO:teuthology.orchestra.run.smithi170.stderr:++ rpm -qa python3-devel 2023-09-22T13:10:39.329 INFO:teuthology.orchestra.run.smithi170.stderr:+ '[' '' == '' ']' 2023-09-22T13:10:39.329 INFO:teuthology.orchestra.run.smithi170.stderr:+ missing=python3-devel 2023-09-22T13:10:39.330 INFO:teuthology.orchestra.run.smithi170.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:10:39.330 INFO:teuthology.orchestra.run.smithi170.stderr:++ rpm -qa libevent-devel 2023-09-22T13:10:40.511 INFO:teuthology.orchestra.run.smithi170.stderr:+ '[' '' == '' ']' 2023-09-22T13:10:40.511 INFO:teuthology.orchestra.run.smithi170.stderr:+ missing='python3-devel libevent-devel' 2023-09-22T13:10:40.511 INFO:teuthology.orchestra.run.smithi170.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:10:40.511 INFO:teuthology.orchestra.run.smithi170.stderr:++ rpm -qa libxml2-devel 2023-09-22T13:10:41.647 INFO:teuthology.orchestra.run.smithi170.stderr:+ '[' '' == '' ']' 2023-09-22T13:10:41.647 INFO:teuthology.orchestra.run.smithi170.stderr:+ missing='python3-devel libevent-devel libxml2-devel' 2023-09-22T13:10:41.647 INFO:teuthology.orchestra.run.smithi170.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:10:41.647 INFO:teuthology.orchestra.run.smithi170.stderr:++ rpm -qa libxslt-devel 2023-09-22T13:10:42.785 INFO:teuthology.orchestra.run.smithi170.stderr:+ '[' '' == '' ']' 2023-09-22T13:10:42.786 INFO:teuthology.orchestra.run.smithi170.stderr:+ missing='python3-devel libevent-devel libxml2-devel libxslt-devel' 2023-09-22T13:10:42.786 INFO:teuthology.orchestra.run.smithi170.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:10:42.786 INFO:teuthology.orchestra.run.smithi170.stderr:++ rpm -qa zlib-devel 2023-09-22T13:10:43.921 INFO:teuthology.orchestra.run.smithi170.stderr:+ '[' '' == '' ']' 2023-09-22T13:10:43.921 INFO:teuthology.orchestra.run.smithi170.stderr:+ missing='python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel' 2023-09-22T13:10:43.922 INFO:teuthology.orchestra.run.smithi170.stderr:+ '[' -n 'python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel' ']' 2023-09-22T13:10:43.922 INFO:teuthology.orchestra.run.smithi170.stderr:+ echo './bootstrap: missing required RPM packages. Installing via sudo.' 2023-09-22T13:10:43.922 INFO:teuthology.orchestra.run.smithi170.stderr:./bootstrap: missing required RPM packages. Installing via sudo. 2023-09-22T13:10:43.922 INFO:teuthology.orchestra.run.smithi170.stderr:+ sudo yum -y install python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:10:44.776 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:29:56 ago on Fri 22 Sep 2023 12:40:48 PM UTC. 2023-09-22T13:10:46.197 INFO:teuthology.orchestra.run.smithi170.stdout:Package python36-devel-3.6.8-38.module_el8.5.0+895+a459eca8.x86_64 is already installed. 2023-09-22T13:10:46.388 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2023-09-22T13:10:46.389 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:10:46.389 INFO:teuthology.orchestra.run.smithi170.stdout: Package Arch Version Repository Size 2023-09-22T13:10:46.389 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:10:46.390 INFO:teuthology.orchestra.run.smithi170.stdout:Installing: 2023-09-22T13:10:46.390 INFO:teuthology.orchestra.run.smithi170.stdout: libevent-devel x86_64 2.1.8-5.el8 appstream 104 k 2023-09-22T13:10:46.390 INFO:teuthology.orchestra.run.smithi170.stdout: libxml2-devel x86_64 2.9.7-16.el8 appstream 1.0 M 2023-09-22T13:10:46.390 INFO:teuthology.orchestra.run.smithi170.stdout: libxslt-devel x86_64 1.1.32-6.el8 appstream 323 k 2023-09-22T13:10:46.390 INFO:teuthology.orchestra.run.smithi170.stdout: zlib-devel x86_64 1.2.11-25.el8 baseos 59 k 2023-09-22T13:10:46.390 INFO:teuthology.orchestra.run.smithi170.stdout:Upgrading: 2023-09-22T13:10:46.390 INFO:teuthology.orchestra.run.smithi170.stdout: zlib x86_64 1.2.11-25.el8 baseos 103 k 2023-09-22T13:10:46.391 INFO:teuthology.orchestra.run.smithi170.stdout:Installing dependencies: 2023-09-22T13:10:46.391 INFO:teuthology.orchestra.run.smithi170.stdout: cmake-filesystem x86_64 3.20.2-5.el8 appstream 45 k 2023-09-22T13:10:46.391 INFO:teuthology.orchestra.run.smithi170.stdout: libgcrypt-devel x86_64 1.8.5-7.el8 baseos 150 k 2023-09-22T13:10:46.391 INFO:teuthology.orchestra.run.smithi170.stdout: libgpg-error-devel x86_64 1.31-1.el8 baseos 64 k 2023-09-22T13:10:46.391 INFO:teuthology.orchestra.run.smithi170.stdout: xz-devel x86_64 5.2.4-4.el8 baseos 63 k 2023-09-22T13:10:46.391 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:10:46.391 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction Summary 2023-09-22T13:10:46.391 INFO:teuthology.orchestra.run.smithi170.stdout:================================================================================ 2023-09-22T13:10:46.392 INFO:teuthology.orchestra.run.smithi170.stdout:Install 8 Packages 2023-09-22T13:10:46.392 INFO:teuthology.orchestra.run.smithi170.stdout:Upgrade 1 Package 2023-09-22T13:10:46.392 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:10:46.392 INFO:teuthology.orchestra.run.smithi170.stdout:Total download size: 1.9 M 2023-09-22T13:10:46.393 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading Packages: 2023-09-22T13:10:47.173 INFO:teuthology.orchestra.run.smithi170.stdout:(1/9): cmake-filesystem-3.20.2-5.el8.x86_64.rpm 116 kB/s | 45 kB 00:00 2023-09-22T13:10:49.565 INFO:teuthology.orchestra.run.smithi170.stdout:(2/9): libevent-devel-2.1.8-5.el8.x86_64.rpm 37 kB/s | 104 kB 00:02 2023-09-22T13:11:08.356 INFO:teuthology.orchestra.run.smithi170.stdout:(3/9): libxslt-devel-1.1.32-6.el8.x86_64.rpm 15 kB/s | 323 kB 00:21 2023-09-22T13:11:42.110 INFO:teuthology.orchestra.run.smithi170.stdout:(4/9): libxml2-devel-2.9.7-16.el8.x86_64.rpm 19 kB/s | 1.0 MB 00:55 2023-09-22T13:12:16.489 INFO:teuthology.orchestra.run.smithi170.stdout:(5/9): xz-devel-5.2.4-4.el8.x86_64.rpm 1.8 kB/s | 63 kB 00:34 2023-09-22T13:12:16.706 INFO:teuthology.orchestra.run.smithi170.stdout:(6/9): zlib-devel-1.2.11-25.el8.x86_64.rpm 270 kB/s | 59 kB 00:00 2023-09-22T13:12:17.090 INFO:teuthology.orchestra.run.smithi170.stdout:(7/9): zlib-1.2.11-25.el8.x86_64.rpm 270 kB/s | 103 kB 00:00 2023-09-22T13:13:00.625 INFO:teuthology.orchestra.run.smithi170.stdout:[MIRROR] libgcrypt-devel-1.8.5-7.el8.x86_64.rpm: Curl error (7): Couldn't connect to server for http://mirrors.unifiedlayer.com/centos/8-stream/BaseOS/x86_64/os/Packages/libgcrypt-devel-1.8.5-7.el8.x86_64.rpm [Failed to connect to mirrors.unifiedlayer.com port 80: Connection timed out] 2023-09-22T13:13:01.218 INFO:teuthology.orchestra.run.smithi170.stdout:(8/9): libgcrypt-devel-1.8.5-7.el8.x86_64.rpm 1.1 kB/s | 150 kB 02:11 2023-09-22T13:13:16.860 INFO:teuthology.orchestra.run.smithi170.stdout:[MIRROR] libgpg-error-devel-1.31-1.el8.x86_64.rpm: Curl error (7): Couldn't connect to server for http://mirrors.unifiedlayer.com/centos/8-stream/BaseOS/x86_64/os/Packages/libgpg-error-devel-1.31-1.el8.x86_64.rpm [Failed to connect to mirrors.unifiedlayer.com port 80: Connection timed out] 2023-09-22T13:13:16.942 INFO:teuthology.orchestra.run.smithi170.stdout:(9/9): libgpg-error-devel-1.31-1.el8.x86_64.rpm 508 B/s | 64 kB 02:08 2023-09-22T13:13:16.942 INFO:teuthology.orchestra.run.smithi170.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:13:16.942 INFO:teuthology.orchestra.run.smithi170.stdout:Total 13 kB/s | 1.9 MB 02:30 2023-09-22T13:13:17.027 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction check 2023-09-22T13:13:17.157 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction check succeeded. 2023-09-22T13:13:17.157 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction test 2023-09-22T13:13:17.264 INFO:teuthology.orchestra.run.smithi170.stdout:Transaction test succeeded. 2023-09-22T13:13:17.267 INFO:teuthology.orchestra.run.smithi170.stdout:Running transaction 2023-09-22T13:13:17.630 INFO:teuthology.orchestra.run.smithi170.stdout: Preparing : 1/1 2023-09-22T13:13:17.774 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libgpg-error-devel-1.31-1.el8.x86_64 1/10 2023-09-22T13:13:17.802 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: libgpg-error-devel-1.31-1.el8.x86_64 1/10 2023-09-22T13:13:18.000 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libgcrypt-devel-1.8.5-7.el8.x86_64 2/10 2023-09-22T13:13:18.015 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: libgcrypt-devel-1.8.5-7.el8.x86_64 2/10 2023-09-22T13:13:18.247 INFO:teuthology.orchestra.run.smithi170.stdout: Upgrading : zlib-1.2.11-25.el8.x86_64 3/10 2023-09-22T13:13:18.420 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : zlib-devel-1.2.11-25.el8.x86_64 4/10 2023-09-22T13:13:18.598 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : xz-devel-5.2.4-4.el8.x86_64 5/10 2023-09-22T13:13:19.050 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : cmake-filesystem-3.20.2-5.el8.x86_64 6/10 2023-09-22T13:13:19.362 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libxml2-devel-2.9.7-16.el8.x86_64 7/10 2023-09-22T13:13:19.604 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libxslt-devel-1.1.32-6.el8.x86_64 8/10 2023-09-22T13:13:19.775 INFO:teuthology.orchestra.run.smithi170.stdout: Installing : libevent-devel-2.1.8-5.el8.x86_64 9/10 2023-09-22T13:13:19.924 INFO:teuthology.orchestra.run.smithi170.stdout: Cleanup : zlib-1.2.11-21.el8.x86_64 10/10 2023-09-22T13:13:20.243 INFO:teuthology.orchestra.run.smithi170.stdout: Running scriptlet: zlib-1.2.11-21.el8.x86_64 10/10 2023-09-22T13:13:20.243 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : cmake-filesystem-3.20.2-5.el8.x86_64 1/10 2023-09-22T13:13:20.243 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libevent-devel-2.1.8-5.el8.x86_64 2/10 2023-09-22T13:13:20.244 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libxml2-devel-2.9.7-16.el8.x86_64 3/10 2023-09-22T13:13:20.244 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libxslt-devel-1.1.32-6.el8.x86_64 4/10 2023-09-22T13:13:20.244 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libgcrypt-devel-1.8.5-7.el8.x86_64 5/10 2023-09-22T13:13:20.244 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : libgpg-error-devel-1.31-1.el8.x86_64 6/10 2023-09-22T13:13:20.244 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : xz-devel-5.2.4-4.el8.x86_64 7/10 2023-09-22T13:13:20.244 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : zlib-devel-1.2.11-25.el8.x86_64 8/10 2023-09-22T13:13:20.244 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : zlib-1.2.11-25.el8.x86_64 9/10 2023-09-22T13:13:20.938 INFO:teuthology.orchestra.run.smithi170.stdout: Verifying : zlib-1.2.11-21.el8.x86_64 10/10 2023-09-22T13:13:20.938 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:20.938 INFO:teuthology.orchestra.run.smithi170.stdout:Upgraded: 2023-09-22T13:13:20.939 INFO:teuthology.orchestra.run.smithi170.stdout: zlib-1.2.11-25.el8.x86_64 2023-09-22T13:13:20.939 INFO:teuthology.orchestra.run.smithi170.stdout:Installed: 2023-09-22T13:13:20.939 INFO:teuthology.orchestra.run.smithi170.stdout: cmake-filesystem-3.20.2-5.el8.x86_64 libevent-devel-2.1.8-5.el8.x86_64 2023-09-22T13:13:20.939 INFO:teuthology.orchestra.run.smithi170.stdout: libgcrypt-devel-1.8.5-7.el8.x86_64 libgpg-error-devel-1.31-1.el8.x86_64 2023-09-22T13:13:20.939 INFO:teuthology.orchestra.run.smithi170.stdout: libxml2-devel-2.9.7-16.el8.x86_64 libxslt-devel-1.1.32-6.el8.x86_64 2023-09-22T13:13:20.939 INFO:teuthology.orchestra.run.smithi170.stdout: xz-devel-5.2.4-4.el8.x86_64 zlib-devel-1.2.11-25.el8.x86_64 2023-09-22T13:13:20.939 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:20.939 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2023-09-22T13:13:21.174 INFO:teuthology.orchestra.run.smithi170.stderr:+ sudo pip3 install --upgrade --trusted-host apt-mirror.front.sepia.ceph.com setuptools cffi cachecontrol 2023-09-22T13:13:21.493 INFO:teuthology.orchestra.run.smithi170.stderr:WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead. 2023-09-22T13:13:22.015 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting setuptools 2023-09-22T13:13:22.399 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading https://files.pythonhosted.org/packages/b0/3a/88b210db68e56854d0bcf4b38e165e03be377e13907746f825790f3df5bf/setuptools-59.6.0-py3-none-any.whl (952kB) 2023-09-22T13:13:22.878 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting cffi 2023-09-22T13:13:23.102 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading https://files.pythonhosted.org/packages/3a/12/d6066828014b9ccb2bbb8e1d9dc28872d20669b65aeb4a86806a0757813f/cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (402kB) 2023-09-22T13:13:23.198 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting cachecontrol 2023-09-22T13:13:23.319 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading https://files.pythonhosted.org/packages/72/a2/28e0ef082f7d78253aded97933e1d7b94bab3c5be366e8afd6513de4028e/CacheControl-0.12.14-py2.py3-none-any.whl 2023-09-22T13:13:23.466 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting pycparser (from cffi) 2023-09-22T13:13:23.490 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading https://files.pythonhosted.org/packages/62/d5/5f610ebe421e85889f2e55e33b7f9a6795bd982198517d912eb1c76e1a53/pycparser-2.21-py2.py3-none-any.whl (118kB) 2023-09-22T13:13:23.563 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting msgpack>=0.5.2 (from cachecontrol) 2023-09-22T13:13:23.784 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading https://files.pythonhosted.org/packages/dc/a1/eba11a0d4b764bc62966a565b470f8c6f38242723ba3057e9b5098678c30/msgpack-1.0.5.tar.gz (127kB) 2023-09-22T13:13:24.396 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting requests (from cachecontrol) 2023-09-22T13:13:24.480 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading https://files.pythonhosted.org/packages/2d/61/08076519c80041bc0ffa1a8af0cbd3bf3e2b62af10435d269a9d0f40564d/requests-2.27.1-py2.py3-none-any.whl (63kB) 2023-09-22T13:13:24.659 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting idna<4,>=2.5; python_version >= "3" (from requests->cachecontrol) 2023-09-22T13:13:24.711 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl (61kB) 2023-09-22T13:13:24.893 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting urllib3<1.27,>=1.21.1 (from requests->cachecontrol) 2023-09-22T13:13:24.975 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading https://files.pythonhosted.org/packages/c5/05/c214b32d21c0b465506f95c4f28ccbcba15022e000b043b72b3df7728471/urllib3-1.26.16-py2.py3-none-any.whl (143kB) 2023-09-22T13:13:25.070 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting charset-normalizer~=2.0.0; python_version >= "3" (from requests->cachecontrol) 2023-09-22T13:13:25.330 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading https://files.pythonhosted.org/packages/06/b3/24afc8868eba069a7f03650ac750a778862dc34941a4bebeb58706715726/charset_normalizer-2.0.12-py3-none-any.whl 2023-09-22T13:13:25.521 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting certifi>=2017.4.17 (from requests->cachecontrol) 2023-09-22T13:13:25.561 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl (158kB) 2023-09-22T13:13:25.578 INFO:teuthology.orchestra.run.smithi170.stdout:Installing collected packages: setuptools, pycparser, cffi, msgpack, idna, urllib3, charset-normalizer, certifi, requests, cachecontrol 2023-09-22T13:13:25.940 INFO:teuthology.orchestra.run.smithi170.stdout: Running setup.py install for msgpack: started 2023-09-22T13:13:26.241 INFO:teuthology.orchestra.run.smithi170.stdout: Running setup.py install for msgpack: finished with status 'done' 2023-09-22T13:13:26.428 INFO:teuthology.orchestra.run.smithi170.stdout:Successfully installed cachecontrol-0.12.14 certifi-2023.7.22 cffi-1.15.1 charset-normalizer-2.0.12 idna-3.4 msgpack-1.0.5 pycparser-2.21 requests-2.27.1 setuptools-59.6.0 urllib3-1.26.16 2023-09-22T13:13:26.548 INFO:teuthology.orchestra.run.smithi170.stderr:+ virtualenv -p python3 --system-site-packages --download virtualenv 2023-09-22T13:13:26.907 INFO:teuthology.orchestra.run.smithi170.stdout:Using base prefix '/usr' 2023-09-22T13:13:26.908 INFO:teuthology.orchestra.run.smithi170.stdout:New python executable in /home/ubuntu/cephtest/ragweed/virtualenv/bin/python3 2023-09-22T13:13:26.908 INFO:teuthology.orchestra.run.smithi170.stdout:Also creating executable in /home/ubuntu/cephtest/ragweed/virtualenv/bin/python 2023-09-22T13:13:29.793 INFO:teuthology.orchestra.run.smithi170.stdout:Installing setuptools, pip, wheel...done. 2023-09-22T13:13:38.492 INFO:teuthology.orchestra.run.smithi170.stdout:Running virtualenv with interpreter /usr/bin/python3 2023-09-22T13:13:38.493 INFO:teuthology.orchestra.run.smithi170.stderr:+ ./virtualenv/bin/pip install --upgrade pip 2023-09-22T13:13:38.493 INFO:teuthology.orchestra.run.smithi170.stdout:Requirement already satisfied: pip in ./virtualenv/lib/python3.6/site-packages (21.3.1) 2023-09-22T13:13:38.493 INFO:teuthology.orchestra.run.smithi170.stderr:+ ./virtualenv/bin/pip install six 2023-09-22T13:13:38.493 INFO:teuthology.orchestra.run.smithi170.stdout:Requirement already satisfied: six in /usr/lib/python3.6/site-packages (1.11.0) 2023-09-22T13:13:38.494 INFO:teuthology.orchestra.run.smithi170.stderr:+ ./virtualenv/bin/pip install -I nose 2023-09-22T13:13:38.494 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting nose 2023-09-22T13:13:38.494 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading nose-1.3.7-py3-none-any.whl (154 kB) 2023-09-22T13:13:38.494 INFO:teuthology.orchestra.run.smithi170.stdout:Installing collected packages: nose 2023-09-22T13:13:38.495 INFO:teuthology.orchestra.run.smithi170.stdout:Successfully installed nose-1.3.7 2023-09-22T13:13:38.495 INFO:teuthology.orchestra.run.smithi170.stderr:+ ./virtualenv/bin/pip install setuptools 2023-09-22T13:13:38.495 INFO:teuthology.orchestra.run.smithi170.stdout:Requirement already satisfied: setuptools in ./virtualenv/lib/python3.6/site-packages (59.6.0) 2023-09-22T13:13:38.495 INFO:teuthology.orchestra.run.smithi170.stderr:+ ./virtualenv/bin/pip install -U -r requirements.txt 2023-09-22T13:13:38.499 INFO:teuthology.orchestra.run.smithi170.stdout:Requirement already satisfied: PyYAML in /usr/lib64/python3.6/site-packages (from -r requirements.txt (line 1)) (3.12) 2023-09-22T13:13:38.499 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting PyYAML 2023-09-22T13:13:38.499 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (677 kB) 2023-09-22T13:13:38.499 INFO:teuthology.orchestra.run.smithi170.stdout:Requirement already satisfied: nose>=1.0.0 in ./virtualenv/lib/python3.6/site-packages (from -r requirements.txt (line 2)) (1.3.7) 2023-09-22T13:13:38.500 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting boto!=2.46.0,>=2.6.0 2023-09-22T13:13:38.500 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading boto-2.49.0-py2.py3-none-any.whl (1.4 MB) 2023-09-22T13:13:38.500 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting munch>=1.0.0 2023-09-22T13:13:38.500 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading munch-4.0.0-py2.py3-none-any.whl (9.9 kB) 2023-09-22T13:13:38.500 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting gevent>=1.0 2023-09-22T13:13:38.500 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading gevent-22.10.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.8 MB) 2023-09-22T13:13:38.500 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting httplib2 2023-09-22T13:13:38.500 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading httplib2-0.22.0-py3-none-any.whl (96 kB) 2023-09-22T13:13:38.500 INFO:teuthology.orchestra.run.smithi170.stdout:Requirement already satisfied: lxml in /usr/lib64/python3.6/site-packages (from -r requirements.txt (line 7)) (4.2.3) 2023-09-22T13:13:38.501 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting lxml 2023-09-22T13:13:38.501 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading lxml-4.9.3-cp36-cp36m-manylinux_2_28_x86_64.whl (7.4 MB) 2023-09-22T13:13:38.501 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting importlib-metadata>=1.7.0 2023-09-22T13:13:38.501 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading importlib_metadata-4.8.3-py3-none-any.whl (17 kB) 2023-09-22T13:13:38.501 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting zope.interface 2023-09-22T13:13:38.501 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading zope.interface-5.5.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (253 kB) 2023-09-22T13:13:38.501 INFO:teuthology.orchestra.run.smithi170.stdout:Requirement already satisfied: setuptools in ./virtualenv/lib/python3.6/site-packages (from gevent>=1.0->-r requirements.txt (line 5)) (59.6.0) 2023-09-22T13:13:38.501 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting zope.event 2023-09-22T13:13:38.501 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading zope.event-4.6-py2.py3-none-any.whl (6.8 kB) 2023-09-22T13:13:38.502 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting greenlet>=2.0.0 2023-09-22T13:13:38.502 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading greenlet-2.0.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (564 kB) 2023-09-22T13:13:38.502 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 2023-09-22T13:13:38.502 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading pyparsing-3.1.1-py3-none-any.whl (103 kB) 2023-09-22T13:13:38.502 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting zipp>=0.5 2023-09-22T13:13:38.502 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading zipp-3.6.0-py3-none-any.whl (5.3 kB) 2023-09-22T13:13:38.502 INFO:teuthology.orchestra.run.smithi170.stdout:Collecting typing-extensions>=3.6.4 2023-09-22T13:13:38.502 INFO:teuthology.orchestra.run.smithi170.stdout: Downloading typing_extensions-4.1.1-py3-none-any.whl (26 kB) 2023-09-22T13:13:38.503 INFO:teuthology.orchestra.run.smithi170.stdout:Installing collected packages: zipp, typing-extensions, zope.interface, zope.event, pyparsing, importlib-metadata, greenlet, PyYAML, munch, lxml, httplib2, gevent, boto 2023-09-22T13:13:38.503 INFO:teuthology.orchestra.run.smithi170.stdout: Attempting uninstall: pyparsing 2023-09-22T13:13:38.503 INFO:teuthology.orchestra.run.smithi170.stdout: Found existing installation: pyparsing 2.1.10 2023-09-22T13:13:38.503 INFO:teuthology.orchestra.run.smithi170.stdout: Not uninstalling pyparsing at /usr/lib/python3.6/site-packages, outside environment /home/ubuntu/cephtest/ragweed/virtualenv 2023-09-22T13:13:38.503 INFO:teuthology.orchestra.run.smithi170.stdout: Can't uninstall 'pyparsing'. No files were found to uninstall. 2023-09-22T13:13:38.503 INFO:teuthology.orchestra.run.smithi170.stdout: Attempting uninstall: PyYAML 2023-09-22T13:13:38.503 INFO:teuthology.orchestra.run.smithi170.stdout: Found existing installation: PyYAML 3.12 2023-09-22T13:13:38.503 INFO:teuthology.orchestra.run.smithi170.stdout: Not uninstalling pyyaml at /usr/lib64/python3.6/site-packages, outside environment /home/ubuntu/cephtest/ragweed/virtualenv 2023-09-22T13:13:38.503 INFO:teuthology.orchestra.run.smithi170.stdout: Can't uninstall 'PyYAML'. No files were found to uninstall. 2023-09-22T13:13:38.504 INFO:teuthology.orchestra.run.smithi170.stdout: Attempting uninstall: lxml 2023-09-22T13:13:38.504 INFO:teuthology.orchestra.run.smithi170.stdout: Found existing installation: lxml 4.2.3 2023-09-22T13:13:38.504 INFO:teuthology.orchestra.run.smithi170.stdout: Not uninstalling lxml at /usr/lib64/python3.6/site-packages, outside environment /home/ubuntu/cephtest/ragweed/virtualenv 2023-09-22T13:13:38.504 INFO:teuthology.orchestra.run.smithi170.stdout: Can't uninstall 'lxml'. No files were found to uninstall. 2023-09-22T13:13:39.367 INFO:teuthology.orchestra.run.smithi170.stdout:Successfully installed PyYAML-6.0.1 boto-2.49.0 gevent-22.10.2 greenlet-2.0.2 httplib2-0.22.0 importlib-metadata-4.8.3 lxml-4.9.3 munch-4.0.0 pyparsing-3.1.1 typing-extensions-4.1.1 zipp-3.6.0 zope.event-4.6 zope.interface-5.5.2 2023-09-22T13:13:39.463 INFO:teuthology.orchestra.run.smithi170.stderr:+ ./virtualenv/bin/python setup.py develop 2023-09-22T13:13:39.702 INFO:teuthology.orchestra.run.smithi170.stdout:running develop 2023-09-22T13:13:39.739 INFO:teuthology.orchestra.run.smithi170.stdout:running egg_info 2023-09-22T13:13:39.739 INFO:teuthology.orchestra.run.smithi170.stdout:creating ragweed.egg-info 2023-09-22T13:13:39.740 INFO:teuthology.orchestra.run.smithi170.stdout:writing ragweed.egg-info/PKG-INFO 2023-09-22T13:13:39.740 INFO:teuthology.orchestra.run.smithi170.stdout:writing dependency_links to ragweed.egg-info/dependency_links.txt 2023-09-22T13:13:39.740 INFO:teuthology.orchestra.run.smithi170.stdout:writing requirements to ragweed.egg-info/requires.txt 2023-09-22T13:13:39.740 INFO:teuthology.orchestra.run.smithi170.stdout:writing top-level names to ragweed.egg-info/top_level.txt 2023-09-22T13:13:39.740 INFO:teuthology.orchestra.run.smithi170.stdout:writing manifest file 'ragweed.egg-info/SOURCES.txt' 2023-09-22T13:13:39.740 INFO:teuthology.orchestra.run.smithi170.stdout:reading manifest file 'ragweed.egg-info/SOURCES.txt' 2023-09-22T13:13:39.741 INFO:teuthology.orchestra.run.smithi170.stdout:adding license file 'LICENSE' 2023-09-22T13:13:39.741 INFO:teuthology.orchestra.run.smithi170.stdout:writing manifest file 'ragweed.egg-info/SOURCES.txt' 2023-09-22T13:13:39.742 INFO:teuthology.orchestra.run.smithi170.stdout:running build_ext 2023-09-22T13:13:39.743 INFO:teuthology.orchestra.run.smithi170.stdout:Creating /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/ragweed.egg-link (link to .) 2023-09-22T13:13:39.743 INFO:teuthology.orchestra.run.smithi170.stdout:Adding ragweed 0.0.1 to easy-install.pth file 2023-09-22T13:13:39.743 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:39.743 INFO:teuthology.orchestra.run.smithi170.stdout:Installed /home/ubuntu/cephtest/ragweed 2023-09-22T13:13:39.744 INFO:teuthology.orchestra.run.smithi170.stdout:Processing dependencies for ragweed==0.0.1 2023-09-22T13:13:39.747 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for isodate>=0.4.4 2023-09-22T13:13:39.747 INFO:teuthology.orchestra.run.smithi170.stdout:Reading https://pypi.org/simple/isodate/ 2023-09-22T13:13:39.834 INFO:teuthology.orchestra.run.smithi170.stdout:Downloading https://files.pythonhosted.org/packages/b6/85/7882d311924cbcfc70b1890780763e36ff0b140c7e51c110fc59a532f087/isodate-0.6.1-py2.py3-none-any.whl#sha256=0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96 2023-09-22T13:13:39.915 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: isodate 0.6.1 2023-09-22T13:13:39.915 INFO:teuthology.orchestra.run.smithi170.stdout:Processing isodate-0.6.1-py2.py3-none-any.whl 2023-09-22T13:13:39.921 INFO:teuthology.orchestra.run.smithi170.stdout:Installing isodate-0.6.1-py2.py3-none-any.whl to /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:13:39.928 INFO:teuthology.orchestra.run.smithi170.stdout:Adding isodate 0.6.1 to easy-install.pth file 2023-09-22T13:13:39.929 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:39.929 INFO:teuthology.orchestra.run.smithi170.stdout:Installed /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/isodate-0.6.1-py3.6.egg 2023-09-22T13:13:40.034 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for gevent==22.10.2 2023-09-22T13:13:40.035 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: gevent 22.10.2 2023-09-22T13:13:40.035 INFO:teuthology.orchestra.run.smithi170.stdout:Adding gevent 22.10.2 to easy-install.pth file 2023-09-22T13:13:40.035 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.035 INFO:teuthology.orchestra.run.smithi170.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:13:40.036 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for munch==4.0.0 2023-09-22T13:13:40.036 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: munch 4.0.0 2023-09-22T13:13:40.037 INFO:teuthology.orchestra.run.smithi170.stdout:Adding munch 4.0.0 to easy-install.pth file 2023-09-22T13:13:40.037 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.037 INFO:teuthology.orchestra.run.smithi170.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:13:40.037 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for PyYAML==6.0.1 2023-09-22T13:13:40.037 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: PyYAML 6.0.1 2023-09-22T13:13:40.038 INFO:teuthology.orchestra.run.smithi170.stdout:Adding PyYAML 6.0.1 to easy-install.pth file 2023-09-22T13:13:40.038 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.038 INFO:teuthology.orchestra.run.smithi170.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:13:40.040 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for boto==2.49.0 2023-09-22T13:13:40.041 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: boto 2.49.0 2023-09-22T13:13:40.041 INFO:teuthology.orchestra.run.smithi170.stdout:Adding boto 2.49.0 to easy-install.pth file 2023-09-22T13:13:40.041 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.041 INFO:teuthology.orchestra.run.smithi170.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:13:40.042 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for six==1.11.0 2023-09-22T13:13:40.042 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: six 1.11.0 2023-09-22T13:13:40.042 INFO:teuthology.orchestra.run.smithi170.stdout:Adding six 1.11.0 to easy-install.pth file 2023-09-22T13:13:40.042 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.042 INFO:teuthology.orchestra.run.smithi170.stdout:Using /usr/lib/python3.6/site-packages 2023-09-22T13:13:40.043 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for zope.event==4.6 2023-09-22T13:13:40.043 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: zope.event 4.6 2023-09-22T13:13:40.043 INFO:teuthology.orchestra.run.smithi170.stdout:Adding zope.event 4.6 to easy-install.pth file 2023-09-22T13:13:40.043 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.043 INFO:teuthology.orchestra.run.smithi170.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:13:40.044 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for greenlet==2.0.2 2023-09-22T13:13:40.044 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: greenlet 2.0.2 2023-09-22T13:13:40.044 INFO:teuthology.orchestra.run.smithi170.stdout:Adding greenlet 2.0.2 to easy-install.pth file 2023-09-22T13:13:40.044 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.044 INFO:teuthology.orchestra.run.smithi170.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:13:40.045 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for zope.interface==5.5.2 2023-09-22T13:13:40.045 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: zope.interface 5.5.2 2023-09-22T13:13:40.045 INFO:teuthology.orchestra.run.smithi170.stdout:Adding zope.interface 5.5.2 to easy-install.pth file 2023-09-22T13:13:40.045 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.045 INFO:teuthology.orchestra.run.smithi170.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:13:40.047 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for setuptools==59.6.0 2023-09-22T13:13:40.048 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: setuptools 59.6.0 2023-09-22T13:13:40.048 INFO:teuthology.orchestra.run.smithi170.stdout:Adding setuptools 59.6.0 to easy-install.pth file 2023-09-22T13:13:40.048 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.048 INFO:teuthology.orchestra.run.smithi170.stdout:Using /usr/local/lib/python3.6/site-packages 2023-09-22T13:13:40.049 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for importlib-metadata==4.8.3 2023-09-22T13:13:40.049 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: importlib-metadata 4.8.3 2023-09-22T13:13:40.049 INFO:teuthology.orchestra.run.smithi170.stdout:Adding importlib-metadata 4.8.3 to easy-install.pth file 2023-09-22T13:13:40.049 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.049 INFO:teuthology.orchestra.run.smithi170.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:13:40.050 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for typing-extensions==4.1.1 2023-09-22T13:13:40.050 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: typing-extensions 4.1.1 2023-09-22T13:13:40.050 INFO:teuthology.orchestra.run.smithi170.stdout:Adding typing-extensions 4.1.1 to easy-install.pth file 2023-09-22T13:13:40.050 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.051 INFO:teuthology.orchestra.run.smithi170.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:13:40.051 INFO:teuthology.orchestra.run.smithi170.stdout:Searching for zipp==3.6.0 2023-09-22T13:13:40.051 INFO:teuthology.orchestra.run.smithi170.stdout:Best match: zipp 3.6.0 2023-09-22T13:13:40.051 INFO:teuthology.orchestra.run.smithi170.stdout:Adding zipp 3.6.0 to easy-install.pth file 2023-09-22T13:13:40.051 INFO:teuthology.orchestra.run.smithi170.stdout: 2023-09-22T13:13:40.052 INFO:teuthology.orchestra.run.smithi170.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:13:40.052 INFO:teuthology.orchestra.run.smithi170.stdout:Finished processing dependencies for ragweed==0.0.1 2023-09-22T13:13:40.052 INFO:teuthology.orchestra.run.smithi170.stderr:/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/setuptools/command/easy_install.py:159: EasyInstallDeprecationWarning: easy_install command is deprecated. Use build and pip and other standards-based tools. 2023-09-22T13:13:40.052 INFO:teuthology.orchestra.run.smithi170.stderr: EasyInstallDeprecationWarning, 2023-09-22T13:13:40.052 INFO:teuthology.orchestra.run.smithi170.stderr:/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/setuptools/command/install.py:37: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools. 2023-09-22T13:13:40.053 INFO:teuthology.orchestra.run.smithi170.stderr: setuptools.SetuptoolsDeprecationWarning, 2023-09-22T13:13:40.053 INFO:teuthology.orchestra.run.smithi170.stderr:/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/pkg_resources/__init__.py:119: PkgResourcesDeprecationWarning: is an invalid version and will not be supported in a future release 2023-09-22T13:13:40.053 INFO:teuthology.orchestra.run.smithi170.stderr: PkgResourcesDeprecationWarning, 2023-09-22T13:13:40.067 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:13:40.067 DEBUG:teuthology.orchestra.run.smithi170:> dd of=/home/ubuntu/cephtest/archive/ragweed.client.0.conf 2023-09-22T13:13:40.125 INFO:tasks.ragweed:Configuring boto... 2023-09-22T13:13:40.125 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2023-09-22T13:13:40.126 DEBUG:teuthology.orchestra.run.smithi170:> dd of=/home/ubuntu/cephtest/boto.cfg 2023-09-22T13:13:40.183 DEBUG:teuthology.orchestra.run.smithi170:ragweed tests against rgw> RAGWEED_CONF=/home/ubuntu/cephtest/archive/ragweed.client.0.conf RAGWEED_STAGES=prepare,check BOTO_CONFIG=/home/ubuntu/cephtest/boto.cfg /home/ubuntu/cephtest/ragweed/virtualenv/bin/python -m nose -w /home/ubuntu/cephtest/ragweed -v -a '!fails_on_rgw' 2023-09-22T13:13:43.556 INFO:teuthology.orchestra.run.smithi170.stderr:ragweed.tests.tests.r_test_multipart_defer_complete.test ... ok 2023-09-22T13:13:44.507 INFO:teuthology.orchestra.run.smithi170.stderr:ragweed.tests.tests.r_test_multipart_defer_update_complete.test ... ok 2023-09-22T13:13:45.416 INFO:teuthology.orchestra.run.smithi170.stderr:ragweed.tests.tests.r_test_multipart_index_versioning.test ... ERROR 2023-09-22T13:13:46.376 INFO:teuthology.orchestra.run.smithi170.stderr:ragweed.tests.tests.r_test_multipart_simple.test ... ok 2023-09-22T13:13:50.697 INFO:teuthology.orchestra.run.smithi170.stderr:ragweed.tests.tests.r_test_obj_storage_class.test ... ok 2023-09-22T13:13:53.545 INFO:teuthology.orchestra.run.smithi170.stderr:ragweed.tests.tests.r_test_obj_storage_class_copy.test ... ok 2023-09-22T13:13:55.601 INFO:teuthology.orchestra.run.smithi170.stderr:ragweed.tests.tests.r_test_obj_storage_class_multipart.test ... ok 2023-09-22T13:13:59.389 INFO:teuthology.orchestra.run.smithi170.stderr:ragweed.tests.tests.r_test_obj_storage_class_multipart_copy.test ... ok 2023-09-22T13:14:02.121 INFO:teuthology.orchestra.run.smithi170.stderr:ragweed.tests.tests.r_test_small_obj_data.test ... ok 2023-09-22T13:14:02.121 INFO:teuthology.orchestra.run.smithi170.stderr: 2023-09-22T13:14:02.121 INFO:teuthology.orchestra.run.smithi170.stderr:====================================================================== 2023-09-22T13:14:02.121 INFO:teuthology.orchestra.run.smithi170.stderr:ERROR: ragweed.tests.tests.r_test_multipart_index_versioning.test 2023-09-22T13:14:02.121 INFO:teuthology.orchestra.run.smithi170.stderr:---------------------------------------------------------------------- 2023-09-22T13:14:02.121 INFO:teuthology.orchestra.run.smithi170.stderr:Traceback (most recent call last): 2023-09-22T13:14:02.122 INFO:teuthology.orchestra.run.smithi170.stderr: File "/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest 2023-09-22T13:14:02.122 INFO:teuthology.orchestra.run.smithi170.stderr: self.test(*self.arg) 2023-09-22T13:14:02.122 INFO:teuthology.orchestra.run.smithi170.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/framework.py", line 345, in test 2023-09-22T13:14:02.122 INFO:teuthology.orchestra.run.smithi170.stderr: self.check() 2023-09-22T13:14:02.122 INFO:teuthology.orchestra.run.smithi170.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/tests/tests.py", line 296, in check 2023-09-22T13:14:02.122 INFO:teuthology.orchestra.run.smithi170.stderr: index_check_result = rgwa().check_bucket_index(rb.name) 2023-09-22T13:14:02.122 INFO:teuthology.orchestra.run.smithi170.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/framework.py", line 68, in check_bucket_index 2023-09-22T13:14:02.123 INFO:teuthology.orchestra.run.smithi170.stderr: return self.get_resource('/admin/bucket',{'index' : None, 'bucket':bucket_name}) 2023-09-22T13:14:02.123 INFO:teuthology.orchestra.run.smithi170.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/framework.py", line 50, in get_resource 2023-09-22T13:14:02.123 INFO:teuthology.orchestra.run.smithi170.stderr: return munch.munchify(json.loads(r.read())) 2023-09-22T13:14:02.123 INFO:teuthology.orchestra.run.smithi170.stderr: File "/usr/lib64/python3.6/json/__init__.py", line 354, in loads 2023-09-22T13:14:02.123 INFO:teuthology.orchestra.run.smithi170.stderr: return _default_decoder.decode(s) 2023-09-22T13:14:02.123 INFO:teuthology.orchestra.run.smithi170.stderr: File "/usr/lib64/python3.6/json/decoder.py", line 342, in decode 2023-09-22T13:14:02.123 INFO:teuthology.orchestra.run.smithi170.stderr: raise JSONDecodeError("Extra data", s, end) 2023-09-22T13:14:02.123 INFO:teuthology.orchestra.run.smithi170.stderr:json.decoder.JSONDecodeError: Extra data: line 1 column 3 (char 2) 2023-09-22T13:14:02.124 INFO:teuthology.orchestra.run.smithi170.stderr:-------------------- >> begin captured stdout << --------------------- 2023-09-22T13:14:02.124 INFO:teuthology.orchestra.run.smithi170.stderr:Munch({'key': 'bucket:test-client.0-r-test-multipart-index-versioning.1', 'ver': Munch({'tag': '_3i06_7n9g2t99C0caUj8H99', 'ver': 1}), 'mtime': '2023-09-22T13:13:44.515133Z', 'data': Munch({'bucket': Munch({'name': 'test-client.0-r-test-multipart-index-versioning.1', 'marker': '772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.5', 'bucket_id': '772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.5', 'tenant': '', 'explicit_placement': Munch({'data_pool': '', 'data_extra_pool': '', 'index_pool': ''})}), 'owner': 'ragweed.client.0', 'creation_time': '2023-09-22T13:13:44.510452Z', 'linked': 'true', 'has_bucket_info': 'false'})}) 2023-09-22T13:14:02.124 INFO:teuthology.orchestra.run.smithi170.stderr:bucket_info: Munch({'bucket': Munch({'name': 'test-client.0-r-test-multipart-index-versioning.1', 'marker': '772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.5', 'bucket_id': '772ce3cc-30ad-4ea4-82fa-9614e9a1eee1.4568.5', 'tenant': '', 'explicit_placement': Munch({'data_pool': '', 'data_extra_pool': '', 'index_pool': ''})}), 'creation_time': '2023-09-22T13:13:44.510452Z', 'owner': 'ragweed.client.0', 'flags': 0, 'zonegroup': '060bb046-2656-46be-891f-3e4dd76a5093', 'placement_rule': 'default-placement', 'has_instance_obj': 'false', 'quota': Munch({'enabled': False, 'check_on_raw': False, 'max_size': -1, 'max_size_kb': 0, 'max_objects': -1}), 'num_shards': 11, 'bi_shard_hash_type': 0, 'requester_pays': 'false', 'has_website': 'false', 'swift_versioning': 'false', 'swift_ver_location': '', 'index_type': 0, 'mdsearch_config': [], 'reshard_status': 0, 'new_bucket_instance_id': ''}) 2023-09-22T13:14:02.124 INFO:teuthology.orchestra.run.smithi170.stderr:read_test_data= b'{"r_buckets": [{"__pickle": "ccopy_reg\\n_reconstructor\\np0\\n(cragweed.framework\\nRBucket\\np1\\nc__builtin__\\nobject\\np2\\nNtp3\\nRp4\\n(dp5\\nVzone\\np6\\ng0\\n(cragweed.framework\\nRZone\\np7\\ng2\\nNtp8\\nRp9\\n(dp10\\nVconn\\np11\\ng0\\n(cmunch\\nMunch\\np12\\nc__builtin__\\ndict\\np13\\n(dp14\\nVsystem\\np15\\ng0\\n(cragweed.framework\\nRGWConnection\\np16\\ng2\\nNtp17\\nRp18\\n(dp19\\nVhost\\np20\\nVsmithi170.front.sepia.ceph.com\\np21\\nsVport\\np22\\nL443L\\nsVis_secure\\np23\\nI01\\nsg11\\ng0\\n(cboto.s3.connection\\nS3Connection\\np24\\ng2\\nNtp25\\nRp26\\n(dp27\\nVcalling_format\\np28\\ng0\\n(cboto.s3.connection\\nOrdinaryCallingFormat\\np29\\ng2\\nNtp30\\nRp31\\nsVbucket_class\\np32\\ncboto.s3.bucket\\nBucket\\np33\\nsVanon\\np34\\nI00\\nsVsuppress_consec_slashes\\np35\\nI01\\nsVnum_retries\\np36\\nL6L\\nsg23\\nI01\\nsVhttps_validate_certificates\\np37\\nNsVca_certificates_file\\np38\\nV/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/boto/cacerts/cacerts.txt\\np39\\nsg22\\nL443L\\nsVproxy\\np40\\nNsVproxy_port\\np41\\nNsVproxy_user\\np42\\nNsVproxy_pass\\np43\\nNsVno_proxy\\np44\\nV\\np45\\nsVuse_proxy\\np46\\nI00\\nsVhttp_exceptions\\np47\\n(chttplib\\nHTTPException\\np48\\ncexceptions\\nOSError\\np49\\ncsocket\\ngaierror\\np50\\nchttplib\\nBadStatusLine\\np51\\ntp52\\nsVhttp_unretryable_exceptions\\np53\\n(lp54\\ncboto.https_connection\\nInvalidCertificateException\\np55\\nasVsocket_exception_values\\np56\\n(L4L\\ntp57\\nsVhttps_connection_factory\\np58\\nNsVprotocol\\np59\\nVhttps\\np60\\nsg20\\ng21\\nsVpath\\np61\\nV/\\np62\\nsVdebug\\np63\\nL0L\\nsVhost_header\\np64\\nNsVhttp_connection_kwargs\\np65\\n(dp66\\nVtimeout\\np67\\nL30L\\nssV_provider_type\\np68\\nVaws\\np69\\nsVprovider\\np70\\ng0\\n(cboto.provider\\nProvider\\np71\\ng2\\nNtp72\\nRp73\\n(dp74\\ng20\\nNsg22\\nNsg64\\nNsV_access_key\\np75\\nVQMCABLXIJQRCJAEYJATB\\np76\\nsV_secret_key\\np77\\nV5aNoOneUFXnoHU8M4T6Lf24/NJkJ1Pc+U9xgWRFYVGTsER61Jjh5dg==\\np78\\nsV_security_token\\np79\\nNsVprofile_name\\np80\\nNsVname\\np81\\ng69\\nsVacl_class\\np82\\ncboto.s3.acl\\nPolicy\\np83\\nsVcanned_acls\\np84\\n(lp85\\nVprivate\\np86\\naVpublic-read\\np87\\naVpublic-read-write\\np88\\naVauthenticated-read\\np89\\naVbucket-owner-read\\np90\\naVbucket-owner-full-control\\np91\\naVlog-delivery-write\\np92\\nasV_credential_expiry_time\\np93\\nNsVshared_credentials\\np94\\ng0\\n(cboto.pyami.config\\nConfig\\np95\\ng2\\nNtp96\\nRp97\\n(dp98\\nV_parser\\np99\\ng0\\n(cConfigParser\\nConfigParser\\np100\\ng2\\nNtp101\\nRp102\\n(dp103\\nV_dict\\np104\\nccollections\\nOrderedDict\\np105\\nsV_sections\\np106\\ng105\\n(tRp107\\nsV_defaults\\np108\\ng105\\n(tRp109\\nVworking_dir\\np110\\nV/mnt/pyami\\np111\\nsVdebug\\np112\\nV0\\np113\\nssV_converters\\np114\\ng0\\n(cConfigParser\\nConverterMapping\\np115\\ng2\\nNtp116\\nRp117\\n(dp118\\ng99\\ng102\\nsV_data\\np119\\n(dp120\\nVboolean\\np121\\nNsVfloat\\np122\\nNsVint\\np123\\nNssbsV_proxies\\np124\\ng105\\n(tRp125\\nVDEFAULT\\np126\\ng0\\n(cConfigParser\\nSectionProxy\\np127\\ng2\\nNtp128\\nRp129\\n(dp130\\ng99\\ng102\\nsV_name\\np131\\ng126\\nsVgetboolean\\np132\\ncfunctools\\npartial\\np133\\n(c__builtin__\\ngetattr\\np134\\n(g129\\nVget\\np135\\ntp136\\nRp137\\ntp138\\nRp139\\n(g137\\n(t(dp140\\nV_impl\\np141\\ng134\\n(g102\\ng132\\ntp142\\nRp143\\nsNtp144\\nbsVgetfloat\\np145\\ng133\\n(g134\\n(g129\\ng135\\ntp146\\nRp147\\ntp148\\nRp149\\n(g147\\n(t(dp150\\ng141\\ng134\\n(g102\\ng145\\ntp151\\nRp152\\nsNtp153\\nbsVgetint\\np154\\ng133\\n(g134\\n(g129\\ng135\\ntp155\\nRp156\\ntp157\\nRp158\\n(g156\\n(t(dp159\\ng141\\ng134\\n(g102\\ng154\\ntp160\\nRp161\\nsNtp162\\nbsbssV_delimiters\\np163\\n(V=\\np164\\nV:\\np165\\ntp166\\nsV_optcre\\np167\\ncre\\n_compile\\np168\\n(V\\\\u000a (?P