2023-09-22T11:39:46.693 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-22T11:39:46.808 DEBUG:teuthology.run:Teuthology command: teuthology --name cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi --archive /home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400908 --description rgw/verify/{centos_latest clusters/fixed-2 frontend/beast ignore-pg-availability msgr-failures/few objectstore/filestore-xfs overrides proto/http rgw_pool_type/ec-profile sharding$/{single} striping$/{stripe-greater-than-chunk} tasks/{0-install bucket-check cls ragweed reshard s3tests-java s3tests versioning} validater/lockdep} --verbose --owner scheduled_cfsnyder@teuthology -- /home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400908/orig.config.yaml 2023-09-22T11:39:46.844 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-22T11:39:46.920 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400908 branch: wip-cfsnyder-bucket-check-pacific description: rgw/verify/{centos_latest clusters/fixed-2 frontend/beast ignore-pg-availability msgr-failures/few objectstore/filestore-xfs overrides proto/http rgw_pool_type/ec-profile sharding$/{single} striping$/{stripe-greater-than-chunk} tasks/{0-install bucket-check cls ragweed reshard s3tests-java s3tests versioning} validater/lockdep} email: null first_in_suite: false job_id: '7400908' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: wip-cfsnyder-bucket-check-pacific ceph: conf: client: debug rgw: 20 rgw crypt require ssl: false rgw crypt s3 kms backend: testing rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= rgw lc debug interval: 10 rgw max chunk size: 4194304 rgw obj stripe size: 6291456 setgroup: ceph setuser: ceph global: mon client directed command retry: 5 ms inject socket failures: 5000 osd_max_pg_log_entries: 10 osd_min_pg_log_entries: 10 rgw override bucket index max shards: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 lockdep: true osd: debug ms: 1 debug osd: 20 lockdep: true osd objectstore: filestore osd shutdown pgref assert: true osd sloppy crc: true flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(PG_AVAILABILITY\) - \(POOL_APP_NOT_ENABLED\) - \(OSD_SLOW_PING_TIME sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 osd: osd objectstore: filestore osd sloppy crc: true filestore: true fs: xfs install: ceph: flavor: default sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 rgw: compression type: random ec-data-pool: true erasure_code_profile: crush-failure-domain: osd k: 3 m: 1 name: testprofile frontend: beast storage classes: LUKEWARM, FROZEN s3tests: slow_backend: true selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-cfsnyder-bucket-check-pacific sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 owner: scheduled_cfsnyder@teuthology priority: 101 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - node-exporter.b sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 sleep_before_teardown: 0 suite: rgw suite_branch: wip-cfsnyder-bucket-check-pacific suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 targets: smithi098.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEnG2dpCSqF0hcwU4l2Bou9zeEz6+Sp3++/ejmApHnc/7fEtxPmKzXNnoR3W7l2Wlnu0AptzOM6FtHOEMLGLUKA= smithi169.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOG6FCbq8Q3wdwSRGUW36jmNWVrtbiundCdeWgGKR9ILzOBTbsBqOr1p7aHTZ6nXFncYlVlXSlcuwgzrlu9yxuI= tasks: - install: null - ceph: null - openssl_keys: null - rgw: client.0: null - workunit: clients: client.0: - rgw/run-bucket-check.sh - workunit: clients: client.0: - cls/test_cls_lock.sh - cls/test_cls_log.sh - cls/test_cls_refcount.sh - cls/test_cls_rgw.sh - cls/test_cls_cmpomap.sh - cls/test_cls_2pc_queue.sh - rgw/test_rgw_gc_log.sh - rgw/test_rgw_obj.sh - rgw/test_rgw_throttle.sh - rgw/test_librgw_file.sh - ragweed: client.0: default-branch: ceph-pacific rgw_server: client.0 stages: prepare,check - workunit: clients: client.0: - rgw/run-reshard.sh - s3tests-java: client.0: force-branch: ceph-pacific force-repo: https://github.com/ceph/java_s3tests.git - s3tests: client.0: force-branch: ceph-pacific rgw_server: client.0 - workunit: clients: client.0: - rgw/run-versioning.sh teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-22_11:28:34 tube: smithi user: cfsnyder verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-22T11:39:46.920 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa; will attempt to use it 2023-09-22T11:39:46.921 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa/tasks 2023-09-22T11:39:46.921 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-22T11:39:46.922 INFO:teuthology.task.internal:Checking packages... 2023-09-22T11:39:46.943 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '31c126b7c1c0c06b66d9d19c5bfc51c46b503392' 2023-09-22T11:39:46.943 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-22T11:39:46.943 INFO:teuthology.packaging:ref: None 2023-09-22T11:39:46.943 INFO:teuthology.packaging:tag: None 2023-09-22T11:39:46.943 INFO:teuthology.packaging:branch: wip-cfsnyder-bucket-check-pacific 2023-09-22T11:39:46.943 INFO:teuthology.packaging:sha1: 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T11:39:46.944 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-cfsnyder-bucket-check-pacific 2023-09-22T11:39:47.252 INFO:teuthology.task.internal:Found packages for ceph version 16.2.14-5.g31c126b7 2023-09-22T11:39:47.252 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-22T11:39:47.262 INFO:teuthology.task.internal:no buildpackages task found 2023-09-22T11:39:47.262 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-22T11:39:47.283 INFO:teuthology.task.internal:Saving configuration 2023-09-22T11:39:47.298 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-22T11:39:47.308 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-22T11:39:47.330 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi098.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400908', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-22 11:29:42.390200', 'locked_by': 'scheduled_cfsnyder@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEnG2dpCSqF0hcwU4l2Bou9zeEz6+Sp3++/ejmApHnc/7fEtxPmKzXNnoR3W7l2Wlnu0AptzOM6FtHOEMLGLUKA='} 2023-09-22T11:39:47.354 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi169.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400908', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-22 11:29:42.391532', 'locked_by': 'scheduled_cfsnyder@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOG6FCbq8Q3wdwSRGUW36jmNWVrtbiundCdeWgGKR9ILzOBTbsBqOr1p7aHTZ6nXFncYlVlXSlcuwgzrlu9yxuI='} 2023-09-22T11:39:47.354 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-22T11:39:47.363 INFO:teuthology.task.internal:roles: ubuntu@smithi098.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a'] 2023-09-22T11:39:47.363 INFO:teuthology.task.internal:roles: ubuntu@smithi169.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'node-exporter.b'] 2023-09-22T11:39:47.363 INFO:teuthology.run_tasks:Running task console_log... 2023-09-22T11:39:47.438 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f782167cee0>, signals=[15]) 2023-09-22T11:39:47.438 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-22T11:39:47.445 INFO:teuthology.task.internal:Opening connections... 2023-09-22T11:39:47.445 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi098.front.sepia.ceph.com 2023-09-22T11:39:47.446 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T11:39:47.514 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi169.front.sepia.ceph.com 2023-09-22T11:39:47.515 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T11:39:47.580 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-22T11:39:47.591 DEBUG:teuthology.orchestra.run.smithi098:> uname -m 2023-09-22T11:39:47.614 INFO:teuthology.orchestra.run.smithi098.stdout:x86_64 2023-09-22T11:39:47.614 DEBUG:teuthology.orchestra.run.smithi098:> cat /etc/os-release 2023-09-22T11:39:47.675 INFO:teuthology.orchestra.run.smithi098.stdout:NAME="CentOS Stream" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:VERSION="8" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:ID="centos" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:ID_LIKE="rhel fedora" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:VERSION_ID="8" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:PLATFORM_ID="platform:el8" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:PRETTY_NAME="CentOS Stream 8" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:ANSI_COLOR="0;31" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:HOME_URL="https://centos.org/" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-22T11:39:47.676 INFO:teuthology.orchestra.run.smithi098.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-09-22T11:39:47.677 INFO:teuthology.lock.ops:Updating smithi098.front.sepia.ceph.com on lock server 2023-09-22T11:39:47.706 DEBUG:teuthology.orchestra.run.smithi169:> uname -m 2023-09-22T11:39:47.728 INFO:teuthology.orchestra.run.smithi169.stdout:x86_64 2023-09-22T11:39:47.767 DEBUG:teuthology.orchestra.run.smithi169:> cat /etc/os-release 2023-09-22T11:39:47.790 INFO:teuthology.orchestra.run.smithi169.stdout:NAME="CentOS Stream" 2023-09-22T11:39:47.790 INFO:teuthology.orchestra.run.smithi169.stdout:VERSION="8" 2023-09-22T11:39:47.790 INFO:teuthology.orchestra.run.smithi169.stdout:ID="centos" 2023-09-22T11:39:47.790 INFO:teuthology.orchestra.run.smithi169.stdout:ID_LIKE="rhel fedora" 2023-09-22T11:39:47.790 INFO:teuthology.orchestra.run.smithi169.stdout:VERSION_ID="8" 2023-09-22T11:39:47.790 INFO:teuthology.orchestra.run.smithi169.stdout:PLATFORM_ID="platform:el8" 2023-09-22T11:39:47.790 INFO:teuthology.orchestra.run.smithi169.stdout:PRETTY_NAME="CentOS Stream 8" 2023-09-22T11:39:47.791 INFO:teuthology.orchestra.run.smithi169.stdout:ANSI_COLOR="0;31" 2023-09-22T11:39:47.791 INFO:teuthology.orchestra.run.smithi169.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-09-22T11:39:47.791 INFO:teuthology.orchestra.run.smithi169.stdout:HOME_URL="https://centos.org/" 2023-09-22T11:39:47.791 INFO:teuthology.orchestra.run.smithi169.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-22T11:39:47.791 INFO:teuthology.orchestra.run.smithi169.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-22T11:39:47.791 INFO:teuthology.orchestra.run.smithi169.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-09-22T11:39:47.791 INFO:teuthology.lock.ops:Updating smithi169.front.sepia.ceph.com on lock server 2023-09-22T11:39:47.814 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-22T11:39:47.827 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-22T11:39:47.840 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-22T11:39:47.841 DEBUG:teuthology.orchestra.run.smithi098:> test '!' -e /home/ubuntu/cephtest 2023-09-22T11:39:47.843 DEBUG:teuthology.orchestra.run.smithi169:> test '!' -e /home/ubuntu/cephtest 2023-09-22T11:39:47.864 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-22T11:39:47.872 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-22T11:39:47.872 DEBUG:teuthology.orchestra.run.smithi098:> test -z $(ls -A /var/lib/ceph) 2023-09-22T11:39:47.904 DEBUG:teuthology.orchestra.run.smithi169:> test -z $(ls -A /var/lib/ceph) 2023-09-22T11:39:47.930 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-22T11:39:47.977 INFO:teuthology.run_tasks:Running task kernel... 2023-09-22T11:39:47.999 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-22T11:39:47.999 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-22T11:39:47.999 DEBUG:teuthology.orchestra.run.smithi098:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-22T11:39:48.000 DEBUG:teuthology.orchestra.run.smithi169:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-22T11:39:48.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T11:39:48.019 DEBUG:teuthology.orchestra.run.smithi098:> uname -r 2023-09-22T11:39:48.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T11:39:48.020 DEBUG:teuthology.orchestra.run.smithi169:> uname -r 2023-09-22T11:39:48.079 INFO:teuthology.orchestra.run.smithi098.stdout:4.18.0-489.el8.x86_64 2023-09-22T11:39:48.080 INFO:teuthology.task.kernel:Running kernel on smithi098: 4.18.0-489.el8.x86_64 2023-09-22T11:39:48.080 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum install -y kernel 2023-09-22T11:39:48.081 INFO:teuthology.orchestra.run.smithi169.stdout:4.18.0-489.el8.x86_64 2023-09-22T11:39:48.081 INFO:teuthology.task.kernel:Running kernel on smithi169: 4.18.0-489.el8.x86_64 2023-09-22T11:39:48.082 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum install -y kernel 2023-09-22T11:51:38.854 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - AppStream 48 kB/s | 33 MB 11:48 2023-09-22T11:51:47.288 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - BaseOS 20 MB/s | 47 MB 00:02 2023-09-22T11:51:54.404 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - Extras 45 kB/s | 18 kB 00:00 2023-09-22T11:51:55.088 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - Extras common packages 14 kB/s | 6.9 kB 00:00 2023-09-22T11:51:55.440 INFO:teuthology.orchestra.run.smithi098.stdout:Copr repo for python3-asyncssh owned by ceph 17 kB/s | 3.5 kB 00:00 2023-09-22T11:51:56.320 INFO:teuthology.orchestra.run.smithi098.stdout:Extra Packages for Enterprise Linux 22 MB/s | 16 MB 00:00 2023-09-22T11:52:00.141 INFO:teuthology.orchestra.run.smithi098.stdout:lab-extras 470 kB/s | 24 kB 00:00 2023-09-22T11:52:03.757 INFO:teuthology.orchestra.run.smithi098.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-09-22T11:52:03.757 INFO:teuthology.orchestra.run.smithi098.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-09-22T11:52:04.015 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T11:52:04.015 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout: Package Architecture Version Repository Size 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout: kernel x86_64 4.18.0-513.el8 baseos 10 M 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-core x86_64 4.18.0-513.el8 baseos 43 M 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-modules x86_64 4.18.0-513.el8 baseos 35 M 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout:Install 3 Packages 2023-09-22T11:52:04.016 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T11:52:04.017 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 88 M 2023-09-22T11:52:04.017 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 96 M 2023-09-22T11:52:04.017 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T11:52:05.159 INFO:teuthology.orchestra.run.smithi098.stdout:(1/3): kernel-4.18.0-513.el8.x86_64.rpm 11 MB/s | 10 MB 00:00 2023-09-22T11:52:06.122 INFO:teuthology.orchestra.run.smithi098.stdout:(2/3): kernel-modules-4.18.0-513.el8.x86_64.rpm 19 MB/s | 35 MB 00:01 2023-09-22T11:52:06.682 INFO:teuthology.orchestra.run.smithi098.stdout:(3/3): kernel-core-4.18.0-513.el8.x86_64.rpm 17 MB/s | 43 MB 00:02 2023-09-22T11:52:06.683 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T11:52:06.683 INFO:teuthology.orchestra.run.smithi098.stdout:Total 33 MB/s | 88 MB 00:02 2023-09-22T11:52:07.362 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T11:52:07.784 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T11:52:07.784 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T11:52:09.007 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T11:52:09.043 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T11:52:11.822 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T11:52:12.400 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T11:52:16.010 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T11:52:16.425 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T11:52:19.800 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T11:52:20.115 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T11:52:41.955 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 3/3 2023-09-22T11:52:41.977 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T11:52:42.338 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T11:52:42.339 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : kernel-4.18.0-513.el8.x86_64 1/3 2023-09-22T11:52:42.339 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : kernel-core-4.18.0-513.el8.x86_64 2/3 2023-09-22T11:52:43.099 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T11:52:43.099 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T11:52:43.099 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T11:52:43.100 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-4.18.0-513.el8.x86_64 kernel-core-4.18.0-513.el8.x86_64 2023-09-22T11:52:43.100 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-modules-4.18.0-513.el8.x86_64 2023-09-22T11:52:43.100 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T11:52:43.100 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T11:52:43.256 DEBUG:teuthology.orchestra.run.smithi098:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T11:52:43.471 INFO:teuthology.orchestra.run.smithi098.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T11:52:43.472 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T11:52:43.472 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-513.el8.x86_64 2023-09-22T11:52:43.472 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-09-22T11:52:43.472 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-09-22T11:52:43.472 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-09-22T11:52:43.472 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi098.front.sepia.ceph.com, path=None, version=distro) 2023-09-22T11:52:43.472 DEBUG:teuthology.orchestra.run.smithi098:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T11:52:43.679 INFO:teuthology.orchestra.run.smithi098.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T11:52:43.679 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T11:52:43.680 DEBUG:teuthology.orchestra.run.smithi098:> sudo rpm -qi grub2-tools 2023-09-22T11:52:43.713 INFO:teuthology.orchestra.run.smithi098.stdout:Name : grub2-tools 2023-09-22T11:52:43.713 INFO:teuthology.orchestra.run.smithi098.stdout:Epoch : 1 2023-09-22T11:52:43.713 INFO:teuthology.orchestra.run.smithi098.stdout:Version : 2.02 2023-09-22T11:52:43.713 INFO:teuthology.orchestra.run.smithi098.stdout:Release : 129.el8 2023-09-22T11:52:43.713 INFO:teuthology.orchestra.run.smithi098.stdout:Architecture: x86_64 2023-09-22T11:52:43.713 INFO:teuthology.orchestra.run.smithi098.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-09-22T11:52:43.714 INFO:teuthology.orchestra.run.smithi098.stdout:Group : System Environment/Base 2023-09-22T11:52:43.714 INFO:teuthology.orchestra.run.smithi098.stdout:Size : 9265586 2023-09-22T11:52:43.714 INFO:teuthology.orchestra.run.smithi098.stdout:License : GPLv3+ 2023-09-22T11:52:43.714 INFO:teuthology.orchestra.run.smithi098.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-09-22T11:52:43.714 INFO:teuthology.orchestra.run.smithi098.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-09-22T11:52:43.714 INFO:teuthology.orchestra.run.smithi098.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-09-22T11:52:43.714 INFO:teuthology.orchestra.run.smithi098.stdout:Build Host : kbuilder.bsys.centos.org 2023-09-22T11:52:43.714 INFO:teuthology.orchestra.run.smithi098.stdout:Relocations : (not relocatable) 2023-09-22T11:52:43.714 INFO:teuthology.orchestra.run.smithi098.stdout:Packager : CentOS BuildSystem 2023-09-22T11:52:43.715 INFO:teuthology.orchestra.run.smithi098.stdout:Vendor : CentOS 2023-09-22T11:52:43.715 INFO:teuthology.orchestra.run.smithi098.stdout:URL : http://www.gnu.org/software/grub/ 2023-09-22T11:52:43.715 INFO:teuthology.orchestra.run.smithi098.stdout:Summary : Support tools for GRUB. 2023-09-22T11:52:43.715 INFO:teuthology.orchestra.run.smithi098.stdout:Description : 2023-09-22T11:52:43.715 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T11:52:43.715 INFO:teuthology.orchestra.run.smithi098.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-09-22T11:52:43.715 INFO:teuthology.orchestra.run.smithi098.stdout:customizable bootloader with modular architecture. It supports a rich 2023-09-22T11:52:43.715 INFO:teuthology.orchestra.run.smithi098.stdout:variety of kernel formats, file systems, computer architectures and 2023-09-22T11:52:43.715 INFO:teuthology.orchestra.run.smithi098.stdout:hardware devices. 2023-09-22T11:52:43.716 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T11:52:43.716 INFO:teuthology.orchestra.run.smithi098.stdout:This subpackage provides tools for support of all platforms. 2023-09-22T11:52:43.716 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-09-22T11:52:43.717 INFO:teuthology.task.kernel:Updating grub on smithi098 to boot 4.18.0-513.el8.x86_64 2023-09-22T11:52:43.717 DEBUG:teuthology.orchestra.run.smithi098:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-09-22T11:52:45.435 INFO:teuthology.orchestra.run.smithi098.stderr:Generating grub configuration file ... 2023-09-22T11:52:47.558 INFO:teuthology.orchestra.run.smithi098.stderr:done 2023-09-22T11:52:47.560 DEBUG:teuthology.orchestra.run.smithi098:> mktemp 2023-09-22T11:52:47.583 INFO:teuthology.orchestra.run.smithi098.stdout:/tmp/tmp.ocgMNwLI9s 2023-09-22T11:52:47.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.ocgMNwLI9s 2023-09-22T11:52:47.658 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 0666 /tmp/tmp.ocgMNwLI9s 2023-09-22T11:52:47.827 DEBUG:teuthology.orchestra.remote:smithi098:/tmp/tmp.ocgMNwLI9s is 6KB 2023-09-22T11:52:47.877 DEBUG:teuthology.orchestra.run.smithi098:> rm -fr /tmp/tmp.ocgMNwLI9s 2023-09-22T11:52:47.894 DEBUG:teuthology.orchestra.run.smithi098:> sudo /bin/ls /boot/loader/entries || true 2023-09-22T11:52:47.975 INFO:teuthology.orchestra.run.smithi098.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-09-22T11:52:47.975 INFO:teuthology.orchestra.run.smithi098.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-09-22T11:52:47.975 INFO:teuthology.orchestra.run.smithi098.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-09-22T11:52:47.975 INFO:teuthology.orchestra.run.smithi098.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64.conf 2023-09-22T11:52:47.976 DEBUG:teuthology.orchestra.run.smithi098:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64 2023-09-22T11:52:48.144 DEBUG:teuthology.orchestra.run.smithi098:> sudo shutdown -r now 2023-09-22T11:53:18.148 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-09-22T11:53:18.149 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-22T11:53:18.150 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T11:53:36.634 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.98 2023-09-22T11:53:45.636 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-22T11:53:45.636 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T11:53:48.698 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.98 2023-09-22T11:54:00.699 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-22T11:54:00.700 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T11:54:32.977 DEBUG:teuthology.orchestra.run.smithi098:> true 2023-09-22T11:54:33.256 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-22T11:54:33.256 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-513.el8.x86_64"... 2023-09-22T11:54:33.256 DEBUG:teuthology.orchestra.run.smithi098:> uname -r 2023-09-22T11:54:33.273 INFO:teuthology.orchestra.run.smithi098.stdout:4.18.0-513.el8.x86_64 2023-09-22T11:54:33.273 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-513.el8.x86_64 vs 4.18.0-513.el8.x86_64 2023-09-22T11:54:33.273 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-09-22T11:54:33.273 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-09-22T11:54:34.274 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-22T11:54:34.275 DEBUG:teuthology.orchestra.run.smithi098:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-22T11:54:34.343 INFO:teuthology.orchestra.run.smithi098.stdout:ttyS1 2023-09-22T11:54:34.359 DEBUG:teuthology.parallel:result is None 2023-09-22T12:01:23.797 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 8 - AppStream 26 kB/s | 33 MB 21:33 2023-09-22T12:01:31.697 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 8 - BaseOS 26 MB/s | 47 MB 00:01 2023-09-22T12:01:38.871 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 8 - Extras 65 kB/s | 18 kB 00:00 2023-09-22T12:01:39.247 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 8 - Extras common packages 35 kB/s | 6.9 kB 00:00 2023-09-22T12:01:40.407 INFO:teuthology.orchestra.run.smithi169.stdout:Copr repo for python3-asyncssh owned by ceph 3.4 kB/s | 3.5 kB 00:01 2023-09-22T12:02:58.683 INFO:teuthology.orchestra.run.smithi169.stdout:Extra Packages for Enterprise Linux 208 kB/s | 16 MB 01:18 2023-09-22T12:03:02.508 INFO:teuthology.orchestra.run.smithi169.stdout:lab-extras 244 kB/s | 24 kB 00:00 2023-09-22T12:03:06.096 INFO:teuthology.orchestra.run.smithi169.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-09-22T12:03:06.096 INFO:teuthology.orchestra.run.smithi169.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-09-22T12:03:06.358 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T12:03:06.358 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:03:06.359 INFO:teuthology.orchestra.run.smithi169.stdout: Package Architecture Version Repository Size 2023-09-22T12:03:06.359 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:03:06.359 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T12:03:06.359 INFO:teuthology.orchestra.run.smithi169.stdout: kernel x86_64 4.18.0-513.el8 baseos 10 M 2023-09-22T12:03:06.359 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-09-22T12:03:06.359 INFO:teuthology.orchestra.run.smithi169.stdout: kernel-core x86_64 4.18.0-513.el8 baseos 43 M 2023-09-22T12:03:06.359 INFO:teuthology.orchestra.run.smithi169.stdout: kernel-modules x86_64 4.18.0-513.el8 baseos 35 M 2023-09-22T12:03:06.359 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:03:06.359 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T12:03:06.359 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:03:06.359 INFO:teuthology.orchestra.run.smithi169.stdout:Install 3 Packages 2023-09-22T12:03:06.360 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:03:06.360 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 88 M 2023-09-22T12:03:06.360 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 96 M 2023-09-22T12:03:06.360 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T12:03:08.934 INFO:teuthology.orchestra.run.smithi169.stdout:(1/3): kernel-modules-4.18.0-513.el8.x86_64.rpm 15 MB/s | 35 MB 00:02 2023-09-22T12:03:09.223 INFO:teuthology.orchestra.run.smithi169.stdout:(2/3): kernel-4.18.0-513.el8.x86_64.rpm 3.8 MB/s | 10 MB 00:02 2023-09-22T12:03:11.599 INFO:teuthology.orchestra.run.smithi169.stdout:(3/3): kernel-core-4.18.0-513.el8.x86_64.rpm 8.4 MB/s | 43 MB 00:05 2023-09-22T12:03:11.599 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:03:11.599 INFO:teuthology.orchestra.run.smithi169.stdout:Total 17 MB/s | 88 MB 00:05 2023-09-22T12:03:12.274 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T12:03:12.714 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T12:03:12.714 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T12:03:13.962 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T12:03:14.001 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T12:03:16.778 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T12:03:17.299 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:03:20.932 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:03:21.332 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:03:24.717 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:03:24.987 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:03:46.821 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: kernel-core-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:03:46.843 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:03:47.194 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: kernel-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:03:47.194 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : kernel-4.18.0-513.el8.x86_64 1/3 2023-09-22T12:03:47.194 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : kernel-core-4.18.0-513.el8.x86_64 2/3 2023-09-22T12:03:47.961 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : kernel-modules-4.18.0-513.el8.x86_64 3/3 2023-09-22T12:03:47.961 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:03:47.961 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T12:03:47.961 INFO:teuthology.orchestra.run.smithi169.stdout: kernel-4.18.0-513.el8.x86_64 kernel-core-4.18.0-513.el8.x86_64 2023-09-22T12:03:47.961 INFO:teuthology.orchestra.run.smithi169.stdout: kernel-modules-4.18.0-513.el8.x86_64 2023-09-22T12:03:47.961 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:03:47.961 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T12:03:48.144 DEBUG:teuthology.orchestra.run.smithi169:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T12:03:48.360 INFO:teuthology.orchestra.run.smithi169.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T12:03:48.360 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T12:03:48.360 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-513.el8.x86_64 2023-09-22T12:03:48.360 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-09-22T12:03:48.360 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-09-22T12:03:48.361 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-09-22T12:03:48.361 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi169.front.sepia.ceph.com, path=None, version=distro) 2023-09-22T12:03:48.361 DEBUG:teuthology.orchestra.run.smithi169:> rpm -q kernel | sort -rV | head -n 1 2023-09-22T12:03:48.570 INFO:teuthology.orchestra.run.smithi169.stdout:kernel-4.18.0-513.el8.x86_64 2023-09-22T12:03:48.570 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.el8.x86_64 2023-09-22T12:03:48.570 DEBUG:teuthology.orchestra.run.smithi169:> sudo rpm -qi grub2-tools 2023-09-22T12:03:48.605 INFO:teuthology.orchestra.run.smithi169.stdout:Name : grub2-tools 2023-09-22T12:03:48.605 INFO:teuthology.orchestra.run.smithi169.stdout:Epoch : 1 2023-09-22T12:03:48.605 INFO:teuthology.orchestra.run.smithi169.stdout:Version : 2.02 2023-09-22T12:03:48.605 INFO:teuthology.orchestra.run.smithi169.stdout:Release : 129.el8 2023-09-22T12:03:48.605 INFO:teuthology.orchestra.run.smithi169.stdout:Architecture: x86_64 2023-09-22T12:03:48.605 INFO:teuthology.orchestra.run.smithi169.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-09-22T12:03:48.605 INFO:teuthology.orchestra.run.smithi169.stdout:Group : System Environment/Base 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:Size : 9265586 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:License : GPLv3+ 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:Build Host : kbuilder.bsys.centos.org 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:Relocations : (not relocatable) 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:Packager : CentOS BuildSystem 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:Vendor : CentOS 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:URL : http://www.gnu.org/software/grub/ 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:Summary : Support tools for GRUB. 2023-09-22T12:03:48.606 INFO:teuthology.orchestra.run.smithi169.stdout:Description : 2023-09-22T12:03:48.607 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:03:48.607 INFO:teuthology.orchestra.run.smithi169.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-09-22T12:03:48.607 INFO:teuthology.orchestra.run.smithi169.stdout:customizable bootloader with modular architecture. It supports a rich 2023-09-22T12:03:48.607 INFO:teuthology.orchestra.run.smithi169.stdout:variety of kernel formats, file systems, computer architectures and 2023-09-22T12:03:48.607 INFO:teuthology.orchestra.run.smithi169.stdout:hardware devices. 2023-09-22T12:03:48.607 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:03:48.607 INFO:teuthology.orchestra.run.smithi169.stdout:This subpackage provides tools for support of all platforms. 2023-09-22T12:03:48.608 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-09-22T12:03:48.608 INFO:teuthology.task.kernel:Updating grub on smithi169 to boot 4.18.0-513.el8.x86_64 2023-09-22T12:03:48.608 DEBUG:teuthology.orchestra.run.smithi169:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-09-22T12:03:50.523 INFO:teuthology.orchestra.run.smithi169.stderr:Generating grub configuration file ... 2023-09-22T12:03:53.160 INFO:teuthology.orchestra.run.smithi169.stderr:done 2023-09-22T12:03:53.161 DEBUG:teuthology.orchestra.run.smithi169:> mktemp 2023-09-22T12:03:53.195 INFO:teuthology.orchestra.run.smithi169.stdout:/tmp/tmp.0W3J4T2vDa 2023-09-22T12:03:53.195 DEBUG:teuthology.orchestra.run.smithi169:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.0W3J4T2vDa 2023-09-22T12:03:53.269 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 0666 /tmp/tmp.0W3J4T2vDa 2023-09-22T12:03:53.435 DEBUG:teuthology.orchestra.remote:smithi169:/tmp/tmp.0W3J4T2vDa is 6KB 2023-09-22T12:03:53.482 DEBUG:teuthology.orchestra.run.smithi169:> rm -fr /tmp/tmp.0W3J4T2vDa 2023-09-22T12:03:53.500 DEBUG:teuthology.orchestra.run.smithi169:> sudo /bin/ls /boot/loader/entries || true 2023-09-22T12:03:53.570 INFO:teuthology.orchestra.run.smithi169.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-09-22T12:03:53.571 INFO:teuthology.orchestra.run.smithi169.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-09-22T12:03:53.571 INFO:teuthology.orchestra.run.smithi169.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-09-22T12:03:53.571 INFO:teuthology.orchestra.run.smithi169.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64.conf 2023-09-22T12:03:53.572 DEBUG:teuthology.orchestra.run.smithi169:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-513.el8.x86_64 2023-09-22T12:03:53.734 DEBUG:teuthology.orchestra.run.smithi169:> sudo shutdown -r now 2023-09-22T12:04:23.737 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-09-22T12:04:23.738 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2023-09-22T12:04:23.738 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:04:42.234 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.169 2023-09-22T12:04:51.235 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2023-09-22T12:04:51.235 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:04:54.298 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.169 2023-09-22T12:05:06.299 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2023-09-22T12:05:06.299 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:06:06.302 DEBUG:teuthology.orchestra.remote:timed out 2023-09-22T12:06:21.304 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2023-09-22T12:06:21.305 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:06:21.672 DEBUG:teuthology.orchestra.run.smithi169:> true 2023-09-22T12:06:21.990 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi169.front.sepia.ceph.com' 2023-09-22T12:06:21.991 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-513.el8.x86_64"... 2023-09-22T12:06:21.991 DEBUG:teuthology.orchestra.run.smithi169:> uname -r 2023-09-22T12:06:22.010 INFO:teuthology.orchestra.run.smithi169.stdout:4.18.0-513.el8.x86_64 2023-09-22T12:06:22.011 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-513.el8.x86_64 vs 4.18.0-513.el8.x86_64 2023-09-22T12:06:22.011 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-09-22T12:06:22.011 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-09-22T12:06:23.011 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-22T12:06:23.012 DEBUG:teuthology.orchestra.run.smithi169:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-22T12:06:23.115 INFO:teuthology.orchestra.run.smithi169.stdout:ttyS1 2023-09-22T12:06:23.130 DEBUG:teuthology.parallel:result is None 2023-09-22T12:06:23.130 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-22T12:06:23.145 INFO:teuthology.task.internal:Creating test directory... 2023-09-22T12:06:23.146 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-22T12:06:23.149 DEBUG:teuthology.orchestra.run.smithi169:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-22T12:06:23.171 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-22T12:06:23.177 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-22T12:06:23.188 INFO:teuthology.task.internal:Creating archive directory... 2023-09-22T12:06:23.189 DEBUG:teuthology.orchestra.run.smithi098:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-22T12:06:23.213 DEBUG:teuthology.orchestra.run.smithi169:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-22T12:06:23.251 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-22T12:06:23.259 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-22T12:06:23.259 DEBUG:teuthology.orchestra.run.smithi098:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-22T12:06:23.285 DEBUG:teuthology.orchestra.run.smithi169:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-22T12:06:23.322 INFO:teuthology.orchestra.run.smithi169.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:06:23.334 INFO:teuthology.orchestra.run.smithi169.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:06:23.335 INFO:teuthology.orchestra.run.smithi098.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:06:23.347 INFO:teuthology.orchestra.run.smithi098.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-22T12:06:23.349 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-22T12:06:23.356 INFO:teuthology.task.internal:Configuring sudo... 2023-09-22T12:06:23.356 DEBUG:teuthology.orchestra.run.smithi098:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-22T12:06:23.358 DEBUG:teuthology.orchestra.run.smithi169:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-22T12:06:23.406 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-22T12:06:23.417 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-22T12:06:23.418 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-22T12:06:23.428 DEBUG:teuthology.orchestra.run.smithi169:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-22T12:06:23.465 DEBUG:teuthology.orchestra.run.smithi098:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:06:23.537 DEBUG:teuthology.orchestra.run.smithi098:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:06:23.589 DEBUG:teuthology.orchestra.run.smithi098:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:06:23.674 DEBUG:teuthology.orchestra.run.smithi098:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:06:23.718 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T12:06:23.718 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-22T12:06:23.785 DEBUG:teuthology.orchestra.run.smithi169:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:06:23.832 DEBUG:teuthology.orchestra.run.smithi169:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-22T12:06:23.898 DEBUG:teuthology.orchestra.run.smithi169:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:06:23.948 DEBUG:teuthology.orchestra.run.smithi169:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-22T12:06:23.991 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T12:06:23.991 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-22T12:06:24.059 DEBUG:teuthology.orchestra.run.smithi098:> sudo service rsyslog restart 2023-09-22T12:06:24.062 DEBUG:teuthology.orchestra.run.smithi169:> sudo service rsyslog restart 2023-09-22T12:06:24.146 INFO:teuthology.orchestra.run.smithi098.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-22T12:06:24.164 INFO:teuthology.orchestra.run.smithi169.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-22T12:06:24.459 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-22T12:06:24.467 INFO:teuthology.task.internal:Starting timer... 2023-09-22T12:06:24.467 INFO:teuthology.run_tasks:Running task pcp... 2023-09-22T12:06:24.477 INFO:teuthology.run_tasks:Running task selinux... 2023-09-22T12:06:24.486 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-22T12:06:24.486 DEBUG:teuthology.orchestra.run.smithi098:> sudo service auditd rotate 2023-09-22T12:06:24.568 INFO:teuthology.orchestra.run.smithi098.stdout:Rotating logs: 2023-09-22T12:06:24.569 DEBUG:teuthology.orchestra.run.smithi169:> sudo service auditd rotate 2023-09-22T12:06:24.657 INFO:teuthology.orchestra.run.smithi169.stdout:Rotating logs: 2023-09-22T12:06:24.659 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-22T12:06:24.660 DEBUG:teuthology.orchestra.run.smithi098:> /usr/sbin/getenforce 2023-09-22T12:06:24.679 INFO:teuthology.orchestra.run.smithi098.stdout:Permissive 2023-09-22T12:06:24.679 DEBUG:teuthology.orchestra.run.smithi169:> /usr/sbin/getenforce 2023-09-22T12:06:24.797 INFO:teuthology.orchestra.run.smithi169.stdout:Permissive 2023-09-22T12:06:24.797 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi098.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi169.front.sepia.ceph.com': 'permissive'} 2023-09-22T12:06:24.798 DEBUG:teuthology.orchestra.run.smithi098:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-22T12:06:24.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T12:06:24.826 DEBUG:teuthology.orchestra.run.smithi169:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-22T12:06:24.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T12:06:24.853 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-22T12:06:24.854 DEBUG:teuthology.orchestra.run.smithi098:> sudo /usr/sbin/setenforce permissive 2023-09-22T12:06:24.894 DEBUG:teuthology.orchestra.run.smithi169:> sudo /usr/sbin/setenforce permissive 2023-09-22T12:06:24.923 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-22T12:06:24.933 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-22T12:06:24.937 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-22T12:06:25.049 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-22T12:06:25.068 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-22T12:06:25.069 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi098.front.sepia.ceph.com,smithi169.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-22T12:26:28.866 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi098.front.sepia.ceph.com'), Remote(name='ubuntu@smithi169.front.sepia.ceph.com')] 2023-09-22T12:26:28.867 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-22T12:26:28.868 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:26:28.940 DEBUG:teuthology.orchestra.run.smithi098:> true 2023-09-22T12:26:29.009 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi098.front.sepia.ceph.com' 2023-09-22T12:26:29.009 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2023-09-22T12:26:29.010 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-22T12:26:29.080 DEBUG:teuthology.orchestra.run.smithi169:> true 2023-09-22T12:26:29.151 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi169.front.sepia.ceph.com' 2023-09-22T12:26:29.151 INFO:teuthology.run_tasks:Running task clock... 2023-09-22T12:26:29.163 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-22T12:26:29.163 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-22T12:26:29.164 DEBUG:teuthology.orchestra.run.smithi098:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-22T12:26:29.167 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-22T12:26:29.167 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-22T12:26:29.198 INFO:teuthology.orchestra.run.smithi098.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-22T12:26:29.212 INFO:teuthology.orchestra.run.smithi098.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-22T12:26:29.227 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-22T12:26:29.242 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-22T12:26:29.280 INFO:teuthology.orchestra.run.smithi098.stderr:sudo: ntpd: command not found 2023-09-22T12:26:29.291 INFO:teuthology.orchestra.run.smithi098.stdout:506 Cannot talk to daemon 2023-09-22T12:26:29.305 INFO:teuthology.orchestra.run.smithi098.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-22T12:26:29.311 INFO:teuthology.orchestra.run.smithi169.stderr:sudo: ntpd: command not found 2023-09-22T12:26:29.319 INFO:teuthology.orchestra.run.smithi098.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-22T12:26:29.323 INFO:teuthology.orchestra.run.smithi169.stdout:506 Cannot talk to daemon 2023-09-22T12:26:29.336 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-22T12:26:29.350 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-22T12:26:29.384 INFO:teuthology.orchestra.run.smithi098.stderr:bash: ntpq: command not found 2023-09-22T12:26:29.387 INFO:teuthology.orchestra.run.smithi098.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-22T12:26:29.387 INFO:teuthology.orchestra.run.smithi098.stdout:=============================================================================== 2023-09-22T12:26:29.387 INFO:teuthology.orchestra.run.smithi098.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:26:29.387 INFO:teuthology.orchestra.run.smithi098.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:26:29.387 INFO:teuthology.orchestra.run.smithi098.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:26:29.387 INFO:teuthology.orchestra.run.smithi098.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:26:29.411 INFO:teuthology.orchestra.run.smithi169.stderr:bash: ntpq: command not found 2023-09-22T12:26:29.415 INFO:teuthology.orchestra.run.smithi169.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-22T12:26:29.415 INFO:teuthology.orchestra.run.smithi169.stdout:=============================================================================== 2023-09-22T12:26:29.415 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:26:29.415 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:26:29.416 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:26:29.416 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-22T12:26:29.416 INFO:teuthology.run_tasks:Running task install... 2023-09-22T12:26:29.428 DEBUG:teuthology.task.install:project ceph 2023-09-22T12:26:29.428 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}} 2023-09-22T12:26:29.429 DEBUG:teuthology.task.install:config {'flavor': 'default', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'} 2023-09-22T12:26:29.429 INFO:teuthology.task.install:Using flavor: default 2023-09-22T12:26:29.435 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-22T12:26:29.435 INFO:teuthology.task.install:extra packages: [] 2023-09-22T12:26:29.436 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'tag': None, 'wait_for_package': False} 2023-09-22T12:26:29.436 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T12:26:29.438 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'tag': None, 'wait_for_package': False} 2023-09-22T12:26:29.438 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T12:26:29.577 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/ 2023-09-22T12:26:29.577 INFO:teuthology.task.install.rpm:Package version is 16.2.14-5.g31c126b7 2023-09-22T12:26:29.585 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/ 2023-09-22T12:26:29.585 INFO:teuthology.task.install.rpm:Package version is 16.2.14-5.g31c126b7 2023-09-22T12:26:29.753 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-22T12:26:29.753 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T12:26:29.753 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-22T12:26:29.766 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-cfsnyder-bucket-check-pacific/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-22T12:26:29.766 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T12:26:29.766 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-22T12:26:29.783 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-22T12:26:29.783 DEBUG:teuthology.orchestra.run.smithi098:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-22T12:26:29.795 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-22T12:26:29.795 DEBUG:teuthology.orchestra.run.smithi169:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/31c126b7c1c0c06b66d9d19c5bfc51c46b503392/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-22T12:26:29.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-22T12:26:29.863 DEBUG:teuthology.orchestra.run.smithi169:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-22T12:26:29.928 DEBUG:teuthology.orchestra.run.smithi098:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-22T12:26:29.940 DEBUG:teuthology.orchestra.run.smithi169:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-22T12:26:29.995 INFO:teuthology.orchestra.run.smithi098.stdout:check_obsoletes = 1 2023-09-22T12:26:29.996 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum clean all 2023-09-22T12:26:30.006 INFO:teuthology.orchestra.run.smithi169.stdout:check_obsoletes = 1 2023-09-22T12:26:30.008 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum clean all 2023-09-22T12:26:30.266 INFO:teuthology.orchestra.run.smithi098.stdout:76 files removed 2023-09-22T12:26:30.287 INFO:teuthology.orchestra.run.smithi169.stdout:76 files removed 2023-09-22T12:26:30.294 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph-radosgw 2023-09-22T12:26:30.315 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-radosgw 2023-09-22T12:26:31.016 INFO:teuthology.orchestra.run.smithi098.stdout:ceph packages for x86_64 239 kB/s | 76 kB 00:00 2023-09-22T12:26:31.031 INFO:teuthology.orchestra.run.smithi169.stdout:ceph packages for x86_64 248 kB/s | 76 kB 00:00 2023-09-22T12:26:31.283 INFO:teuthology.orchestra.run.smithi098.stdout:ceph noarch packages 67 kB/s | 15 kB 00:00 2023-09-22T12:26:31.398 INFO:teuthology.orchestra.run.smithi169.stdout:ceph noarch packages 47 kB/s | 15 kB 00:00 2023-09-22T12:26:31.601 INFO:teuthology.orchestra.run.smithi098.stdout:ceph source packages 4.9 kB/s | 1.4 kB 00:00 2023-09-22T12:26:31.715 INFO:teuthology.orchestra.run.smithi169.stdout:ceph source packages 5.0 kB/s | 1.4 kB 00:00 2023-09-22T12:44:59.360 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 8 - AppStream 31 kB/s | 33 MB 18:27 2023-09-22T12:45:09.783 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 8 - BaseOS 11 MB/s | 47 MB 00:04 2023-09-22T12:45:16.991 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 8 - Extras 37 kB/s | 18 kB 00:00 2023-09-22T12:45:17.517 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 8 - Extras common packages 20 kB/s | 6.9 kB 00:00 2023-09-22T12:45:20.493 INFO:teuthology.orchestra.run.smithi169.stdout:CentOS Stream 8 - PowerTools 2.3 MB/s | 6.5 MB 00:02 2023-09-22T12:45:22.260 INFO:teuthology.orchestra.run.smithi169.stdout:Copr repo for python3-asyncssh owned by ceph 7.4 kB/s | 3.5 kB 00:00 2023-09-22T12:45:24.188 INFO:teuthology.orchestra.run.smithi169.stdout:Extra Packages for Enterprise Linux 8.9 MB/s | 16 MB 00:01 2023-09-22T12:45:28.069 INFO:teuthology.orchestra.run.smithi169.stdout:lab-extras 168 kB/s | 24 kB 00:00 2023-09-22T12:45:32.463 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T12:45:32.466 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:45:32.466 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-09-22T12:45:32.466 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:45:32.466 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T12:45:32.466 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-radosgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 12 M 2023-09-22T12:45:32.466 INFO:teuthology.orchestra.run.smithi169.stdout:Upgrading: 2023-09-22T12:45:32.466 INFO:teuthology.orchestra.run.smithi169.stdout: librados2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T12:45:32.466 INFO:teuthology.orchestra.run.smithi169.stdout: librbd1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 4.2 M 2023-09-22T12:45:32.466 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-base x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.1 M 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 24 M 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-selinux x86_64 2:16.2.14-5.g31c126b7.el8 ceph 26 k 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: libcephfs2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 824 k 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: libradosstriper1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 494 k 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: librgw2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.8 M 2023-09-22T12:45:32.467 INFO:teuthology.orchestra.run.smithi169.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout: python3-ceph-argparse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 45 k 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout: python3-ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 99 k 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cephfs x86_64 2:16.2.14-5.g31c126b7.el8 ceph 215 k 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rados x86_64 2:16.2.14-5.g31c126b7.el8 ceph 390 k 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 369 k 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 114 k 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout:Install 21 Packages 2023-09-22T12:45:32.468 INFO:teuthology.orchestra.run.smithi169.stdout:Upgrade 2 Packages 2023-09-22T12:45:32.469 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:45:32.469 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 57 M 2023-09-22T12:45:32.469 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T12:45:34.270 INFO:teuthology.orchestra.run.smithi169.stdout:(1/23): ceph-radosgw-16.2.14-5.g31c126b7.el8.x8 8.4 MB/s | 12 MB 00:01 2023-09-22T12:45:34.312 INFO:teuthology.orchestra.run.smithi169.stdout:(2/23): ceph-selinux-16.2.14-5.g31c126b7.el8.x8 617 kB/s | 26 kB 00:00 2023-09-22T12:45:34.604 INFO:teuthology.orchestra.run.smithi169.stdout:(3/23): ceph-common-16.2.14-5.g31c126b7.el8.x86 14 MB/s | 24 MB 00:01 2023-09-22T12:45:34.638 INFO:teuthology.orchestra.run.smithi169.stdout:(4/23): libcephfs2-16.2.14-5.g31c126b7.el8.x86_ 2.5 MB/s | 824 kB 00:00 2023-09-22T12:45:34.663 INFO:teuthology.orchestra.run.smithi169.stdout:(5/23): libradosstriper1-16.2.14-5.g31c126b7.el 8.2 MB/s | 494 kB 00:00 2023-09-22T12:45:34.705 INFO:teuthology.orchestra.run.smithi169.stdout:(6/23): python3-ceph-argparse-16.2.14-5.g31c126 1.1 MB/s | 45 kB 00:00 2023-09-22T12:45:34.755 INFO:teuthology.orchestra.run.smithi169.stdout:(7/23): python3-ceph-common-16.2.14-5.g31c126b7 1.9 MB/s | 99 kB 00:00 2023-09-22T12:45:34.814 INFO:teuthology.orchestra.run.smithi169.stdout:(8/23): python3-cephfs-16.2.14-5.g31c126b7.el8. 3.6 MB/s | 215 kB 00:00 2023-09-22T12:45:34.872 INFO:teuthology.orchestra.run.smithi169.stdout:(9/23): librgw2-16.2.14-5.g31c126b7.el8.x86_64. 16 MB/s | 3.8 MB 00:00 2023-09-22T12:45:34.898 INFO:teuthology.orchestra.run.smithi169.stdout:(10/23): python3-rados-16.2.14-5.g31c126b7.el8. 4.6 MB/s | 390 kB 00:00 2023-09-22T12:45:34.923 INFO:teuthology.orchestra.run.smithi169.stdout:(11/23): python3-rbd-16.2.14-5.g31c126b7.el8.x8 7.1 MB/s | 369 kB 00:00 2023-09-22T12:45:34.949 INFO:teuthology.orchestra.run.smithi169.stdout:(12/23): python3-rgw-16.2.14-5.g31c126b7.el8.x8 2.2 MB/s | 114 kB 00:00 2023-09-22T12:45:36.249 INFO:teuthology.orchestra.run.smithi169.stdout:(13/23): ceph-base-16.2.14-5.g31c126b7.el8.x86_ 1.8 MB/s | 6.1 MB 00:03 2023-09-22T12:45:37.699 INFO:teuthology.orchestra.run.smithi169.stdout:(14/23): python3-prettytable-0.7.2-14.el8.noarc 30 kB/s | 44 kB 00:01 2023-09-22T12:45:37.982 INFO:teuthology.orchestra.run.smithi169.stdout:(15/23): librabbitmq-0.9.0-4.el8.x86_64.rpm 166 kB/s | 47 kB 00:00 2023-09-22T12:45:38.866 INFO:teuthology.orchestra.run.smithi169.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 347 kB/s | 306 kB 00:00 2023-09-22T12:45:39.017 INFO:teuthology.orchestra.run.smithi169.stdout:(17/23): leveldb-1.22-1.el8.x86_64.rpm 1.2 MB/s | 181 kB 00:00 2023-09-22T12:45:39.150 INFO:teuthology.orchestra.run.smithi169.stdout:(18/23): liboath-2.6.2-3.el8.x86_64.rpm 441 kB/s | 59 kB 00:00 2023-09-22T12:45:39.292 INFO:teuthology.orchestra.run.smithi169.stdout:(19/23): libunwind-1.3.1-3.el8.x86_64.rpm 530 kB/s | 75 kB 00:00 2023-09-22T12:45:40.984 INFO:teuthology.orchestra.run.smithi169.stdout:(20/23): librados2-16.2.14-5.g31c126b7.el8.x86_ 2.2 MB/s | 3.7 MB 00:01 2023-09-22T12:45:42.051 INFO:teuthology.orchestra.run.smithi169.stdout:(21/23): librbd1-16.2.14-5.g31c126b7.el8.x86_64 3.9 MB/s | 4.2 MB 00:01 2023-09-22T12:45:42.075 INFO:teuthology.orchestra.run.smithi169.stdout:(22/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 36 kB/s | 259 kB 00:07 2023-09-22T12:45:42.081 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - AppStream 30 kB/s | 33 MB 19:10 2023-09-22T12:45:46.292 INFO:teuthology.orchestra.run.smithi169.stdout:(23/23): librdkafka-0.11.4-3.el8.x86_64.rpm 31 kB/s | 354 kB 00:11 2023-09-22T12:45:46.295 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:45:46.295 INFO:teuthology.orchestra.run.smithi169.stdout:Total 4.1 MB/s | 57 MB 00:13 2023-09-22T12:45:46.338 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T12:45:46.386 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T12:45:46.386 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T12:45:46.730 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T12:45:46.733 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T12:45:47.771 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T12:45:47.979 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2023-09-22T12:45:48.156 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T12:45:48.541 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T12:45:48.697 INFO:teuthology.orchestra.run.smithi169.stdout: Upgrading : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:45:48.774 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:45:49.013 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T12:45:49.189 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:45:49.587 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:45:49.780 INFO:teuthology.orchestra.run.smithi169.stdout: Upgrading : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:45:49.847 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:45:50.036 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 7/25 2023-09-22T12:45:50.205 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T12:45:50.582 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T12:45:50.730 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T12:45:50.829 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T12:45:51.003 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-09-22T12:45:51.152 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 11/25 2023-09-22T12:45:51.332 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T12:45:51.523 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - BaseOS 14 MB/s | 47 MB 00:03 2023-09-22T12:45:51.539 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T12:45:51.739 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 14/25 2023-09-22T12:45:51.889 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T12:45:51.950 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T12:45:52.155 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-22T12:45:52.330 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2023-09-22T12:45:52.508 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2023-09-22T12:45:52.639 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 19/25 2023-09-22T12:45:55.078 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:45:55.297 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:45:55.993 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:45:56.266 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T12:45:56.357 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T12:45:56.479 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:45:58.823 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - Extras 36 kB/s | 18 kB 00:00 2023-09-22T12:45:59.548 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - Extras common packages 13 kB/s | 6.9 kB 00:00 2023-09-22T12:46:00.891 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - PowerTools 5.4 MB/s | 6.5 MB 00:01 2023-09-22T12:46:02.418 INFO:teuthology.orchestra.run.smithi098.stdout:Copr repo for python3-asyncssh owned by ceph 14 kB/s | 3.5 kB 00:00 2023-09-22T12:46:03.854 INFO:teuthology.orchestra.run.smithi098.stdout:Extra Packages for Enterprise Linux 12 MB/s | 16 MB 00:01 2023-09-22T12:46:07.719 INFO:teuthology.orchestra.run.smithi098.stdout:lab-extras 240 kB/s | 24 kB 00:00 2023-09-22T12:46:12.171 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T12:46:12.174 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:46:12.174 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repository Size 2023-09-22T12:46:12.174 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:46:12.174 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T12:46:12.174 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-radosgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 12 M 2023-09-22T12:46:12.174 INFO:teuthology.orchestra.run.smithi098.stdout:Upgrading: 2023-09-22T12:46:12.174 INFO:teuthology.orchestra.run.smithi098.stdout: librados2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T12:46:12.175 INFO:teuthology.orchestra.run.smithi098.stdout: librbd1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 4.2 M 2023-09-22T12:46:12.175 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2023-09-22T12:46:12.175 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-base x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.1 M 2023-09-22T12:46:12.175 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 24 M 2023-09-22T12:46:12.175 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-selinux x86_64 2:16.2.14-5.g31c126b7.el8 ceph 26 k 2023-09-22T12:46:12.175 INFO:teuthology.orchestra.run.smithi098.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-22T12:46:12.175 INFO:teuthology.orchestra.run.smithi098.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-09-22T12:46:12.175 INFO:teuthology.orchestra.run.smithi098.stdout: libcephfs2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 824 k 2023-09-22T12:46:12.175 INFO:teuthology.orchestra.run.smithi098.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-22T12:46:12.175 INFO:teuthology.orchestra.run.smithi098.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-09-22T12:46:12.176 INFO:teuthology.orchestra.run.smithi098.stdout: libradosstriper1 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 494 k 2023-09-22T12:46:12.176 INFO:teuthology.orchestra.run.smithi098.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-09-22T12:46:12.176 INFO:teuthology.orchestra.run.smithi098.stdout: librgw2 x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.8 M 2023-09-22T12:46:12.176 INFO:teuthology.orchestra.run.smithi098.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-22T12:46:12.176 INFO:teuthology.orchestra.run.smithi098.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-09-22T12:46:12.176 INFO:teuthology.orchestra.run.smithi098.stdout: python3-ceph-argparse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 45 k 2023-09-22T12:46:12.176 INFO:teuthology.orchestra.run.smithi098.stdout: python3-ceph-common x86_64 2:16.2.14-5.g31c126b7.el8 ceph 99 k 2023-09-22T12:46:12.176 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cephfs x86_64 2:16.2.14-5.g31c126b7.el8 ceph 215 k 2023-09-22T12:46:12.176 INFO:teuthology.orchestra.run.smithi098.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-09-22T12:46:12.176 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rados x86_64 2:16.2.14-5.g31c126b7.el8 ceph 390 k 2023-09-22T12:46:12.176 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 369 k 2023-09-22T12:46:12.177 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rgw x86_64 2:16.2.14-5.g31c126b7.el8 ceph 114 k 2023-09-22T12:46:12.177 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:46:12.177 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T12:46:12.177 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:46:12.177 INFO:teuthology.orchestra.run.smithi098.stdout:Install 21 Packages 2023-09-22T12:46:12.177 INFO:teuthology.orchestra.run.smithi098.stdout:Upgrade 2 Packages 2023-09-22T12:46:12.177 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:46:12.178 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 57 M 2023-09-22T12:46:12.178 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T12:46:13.795 INFO:teuthology.orchestra.run.smithi098.stdout:(1/23): ceph-base-16.2.14-5.g31c126b7.el8.x86_6 6.7 MB/s | 6.1 MB 00:00 2023-09-22T12:46:13.853 INFO:teuthology.orchestra.run.smithi098.stdout:(2/23): ceph-selinux-16.2.14-5.g31c126b7.el8.x8 442 kB/s | 26 kB 00:00 2023-09-22T12:46:13.987 INFO:teuthology.orchestra.run.smithi098.stdout:(3/23): libcephfs2-16.2.14-5.g31c126b7.el8.x86_ 6.0 MB/s | 824 kB 00:00 2023-09-22T12:46:14.071 INFO:teuthology.orchestra.run.smithi098.stdout:(4/23): libradosstriper1-16.2.14-5.g31c126b7.el 5.8 MB/s | 494 kB 00:00 2023-09-22T12:46:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:46:14.401 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /sys 2023-09-22T12:46:14.401 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /proc 2023-09-22T12:46:14.401 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /dev 2023-09-22T12:46:14.401 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /run 2023-09-22T12:46:14.401 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /mnt 2023-09-22T12:46:14.401 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /var/tmp 2023-09-22T12:46:14.401 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /home 2023-09-22T12:46:14.401 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /tmp 2023-09-22T12:46:14.401 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /dev 2023-09-22T12:46:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:46:14.446 INFO:teuthology.orchestra.run.smithi098.stdout:(5/23): librgw2-16.2.14-5.g31c126b7.el8.x86_64. 10 MB/s | 3.8 MB 00:00 2023-09-22T12:46:14.513 INFO:teuthology.orchestra.run.smithi098.stdout:(6/23): python3-ceph-argparse-16.2.14-5.g31c126 675 kB/s | 45 kB 00:00 2023-09-22T12:46:14.638 INFO:teuthology.orchestra.run.smithi098.stdout:(7/23): ceph-radosgw-16.2.14-5.g31c126b7.el8.x8 6.7 MB/s | 12 MB 00:01 2023-09-22T12:46:14.655 INFO:teuthology.orchestra.run.smithi098.stdout:(8/23): python3-ceph-common-16.2.14-5.g31c126b7 695 kB/s | 99 kB 00:00 2023-09-22T12:46:14.697 INFO:teuthology.orchestra.run.smithi098.stdout:(9/23): python3-cephfs-16.2.14-5.g31c126b7.el8. 3.6 MB/s | 215 kB 00:00 2023-09-22T12:46:14.722 INFO:teuthology.orchestra.run.smithi098.stdout:(10/23): python3-rados-16.2.14-5.g31c126b7.el8. 5.7 MB/s | 390 kB 00:00 2023-09-22T12:46:14.756 INFO:teuthology.orchestra.run.smithi098.stdout:(11/23): python3-rbd-16.2.14-5.g31c126b7.el8.x8 6.1 MB/s | 369 kB 00:00 2023-09-22T12:46:14.781 INFO:teuthology.orchestra.run.smithi098.stdout:(12/23): python3-rgw-16.2.14-5.g31c126b7.el8.x8 1.9 MB/s | 114 kB 00:00 2023-09-22T12:46:15.182 INFO:teuthology.orchestra.run.smithi098.stdout:(13/23): ceph-common-16.2.14-5.g31c126b7.el8.x8 11 MB/s | 24 MB 00:02 2023-09-22T12:46:15.538 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T12:46:15.605 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T12:46:15.605 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T12:46:15.837 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T12:46:15.837 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:46:16.559 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:46:16.560 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 1/25 2023-09-22T12:46:16.560 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2/25 2023-09-22T12:46:16.560 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:46:16.560 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T12:46:16.560 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:46:16.561 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:46:16.561 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 7/25 2023-09-22T12:46:16.561 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 8/25 2023-09-22T12:46:16.561 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 9/25 2023-09-22T12:46:16.561 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 10/25 2023-09-22T12:46:16.561 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 11/25 2023-09-22T12:46:16.561 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T12:46:16.562 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T12:46:16.562 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2023-09-22T12:46:16.562 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2023-09-22T12:46:16.562 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-22T12:46:16.562 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 17/25 2023-09-22T12:46:16.562 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-22T12:46:16.563 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : leveldb-1.22-1.el8.x86_64 19/25 2023-09-22T12:46:16.563 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 20/25 2023-09-22T12:46:16.563 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 21/25 2023-09-22T12:46:16.563 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:46:16.563 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-22T12:46:16.563 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 24/25 2023-09-22T12:46:17.356 INFO:teuthology.orchestra.run.smithi098.stdout:(14/23): python3-prettytable-0.7.2-14.el8.noarc 20 kB/s | 44 kB 00:02 2023-09-22T12:46:17.559 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:46:17.559 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:46:17.559 INFO:teuthology.orchestra.run.smithi169.stdout:Upgraded: 2023-09-22T12:46:17.559 INFO:teuthology.orchestra.run.smithi169.stdout: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.560 INFO:teuthology.orchestra.run.smithi169.stdout: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.560 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T12:46:17.560 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.560 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.560 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.560 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.560 INFO:teuthology.orchestra.run.smithi169.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-22T12:46:17.560 INFO:teuthology.orchestra.run.smithi169.stdout: leveldb-1.22-1.el8.x86_64 2023-09-22T12:46:17.560 INFO:teuthology.orchestra.run.smithi169.stdout: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.561 INFO:teuthology.orchestra.run.smithi169.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-22T12:46:17.561 INFO:teuthology.orchestra.run.smithi169.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-09-22T12:46:17.561 INFO:teuthology.orchestra.run.smithi169.stdout: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.561 INFO:teuthology.orchestra.run.smithi169.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-22T12:46:17.561 INFO:teuthology.orchestra.run.smithi169.stdout: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.561 INFO:teuthology.orchestra.run.smithi169.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-22T12:46:17.561 INFO:teuthology.orchestra.run.smithi169.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-22T12:46:17.561 INFO:teuthology.orchestra.run.smithi169.stdout: python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.561 INFO:teuthology.orchestra.run.smithi169.stdout: python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.562 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.562 INFO:teuthology.orchestra.run.smithi169.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-22T12:46:17.562 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.562 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.562 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:46:17.562 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:46:17.562 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T12:46:17.665 INFO:teuthology.orchestra.run.smithi098.stdout:(15/23): librabbitmq-0.9.0-4.el8.x86_64.rpm 152 kB/s | 47 kB 00:00 2023-09-22T12:46:17.821 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-test 2023-09-22T12:46:18.115 INFO:teuthology.orchestra.run.smithi098.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 680 kB/s | 306 kB 00:00 2023-09-22T12:46:18.199 INFO:teuthology.orchestra.run.smithi098.stdout:(17/23): leveldb-1.22-1.el8.x86_64.rpm 2.1 MB/s | 181 kB 00:00 2023-09-22T12:46:18.291 INFO:teuthology.orchestra.run.smithi098.stdout:(18/23): liboath-2.6.2-3.el8.x86_64.rpm 642 kB/s | 59 kB 00:00 2023-09-22T12:46:18.374 INFO:teuthology.orchestra.run.smithi098.stdout:(19/23): libunwind-1.3.1-3.el8.x86_64.rpm 900 kB/s | 75 kB 00:00 2023-09-22T12:46:18.669 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:00:51 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T12:46:18.700 INFO:teuthology.orchestra.run.smithi098.stdout:(20/23): librados2-16.2.14-5.g31c126b7.el8.x86_ 12 MB/s | 3.7 MB 00:00 2023-09-22T12:46:18.926 INFO:teuthology.orchestra.run.smithi098.stdout:(21/23): librbd1-16.2.14-5.g31c126b7.el8.x86_64 19 MB/s | 4.2 MB 00:00 2023-09-22T12:46:20.257 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T12:46:20.258 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:46:20.258 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-09-22T12:46:20.258 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:46:20.259 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T12:46:20.259 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-test x86_64 2:16.2.14-5.g31c126b7.el8 ceph 52 M 2023-09-22T12:46:20.259 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-09-22T12:46:20.259 INFO:teuthology.orchestra.run.smithi169.stdout: jq x86_64 1.6-7.el8 appstream 202 k 2023-09-22T12:46:20.259 INFO:teuthology.orchestra.run.smithi169.stdout: libcephsqlite x86_64 2:16.2.14-5.g31c126b7.el8 ceph 174 k 2023-09-22T12:46:20.259 INFO:teuthology.orchestra.run.smithi169.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-09-22T12:46:20.259 INFO:teuthology.orchestra.run.smithi169.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-09-22T12:46:20.259 INFO:teuthology.orchestra.run.smithi169.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-09-22T12:46:20.259 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:46:20.259 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T12:46:20.259 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:46:20.260 INFO:teuthology.orchestra.run.smithi169.stdout:Install 6 Packages 2023-09-22T12:46:20.260 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:46:20.260 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 53 M 2023-09-22T12:46:20.260 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 196 M 2023-09-22T12:46:20.260 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T12:46:20.921 INFO:teuthology.orchestra.run.smithi169.stdout:(1/6): libcephsqlite-16.2.14-5.g31c126b7.el8.x8 384 kB/s | 174 kB 00:00 2023-09-22T12:46:22.938 INFO:teuthology.orchestra.run.smithi169.stdout:(2/6): ceph-test-16.2.14-5.g31c126b7.el8.x86_64 21 MB/s | 52 MB 00:02 2023-09-22T12:46:24.508 INFO:teuthology.orchestra.run.smithi098.stdout:(22/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 27 kB/s | 259 kB 00:09 2023-09-22T12:46:24.863 INFO:teuthology.orchestra.run.smithi169.stdout:(3/6): jq-1.6-7.el8.x86_64.rpm 46 kB/s | 202 kB 00:04 2023-09-22T12:46:25.430 INFO:teuthology.orchestra.run.smithi169.stdout:(4/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 125 kB/s | 71 kB 00:00 2023-09-22T12:46:25.989 INFO:teuthology.orchestra.run.smithi169.stdout:(5/6): oniguruma-6.8.2-2.el8.x86_64.rpm 37 kB/s | 187 kB 00:05 2023-09-22T12:46:30.892 INFO:teuthology.orchestra.run.smithi098.stdout:(23/23): librdkafka-0.11.4-3.el8.x86_64.rpm 22 kB/s | 354 kB 00:16 2023-09-22T12:46:30.894 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:46:30.894 INFO:teuthology.orchestra.run.smithi098.stdout:Total 3.1 MB/s | 57 MB 00:18 2023-09-22T12:46:30.935 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T12:46:30.983 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T12:46:30.983 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T12:46:31.327 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T12:46:31.330 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T12:46:32.359 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T12:46:32.513 INFO:teuthology.orchestra.run.smithi169.stdout:(6/6): socat-1.7.4.1-1.el8.x86_64.rpm 34 kB/s | 323 kB 00:09 2023-09-22T12:46:32.514 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:46:32.514 INFO:teuthology.orchestra.run.smithi169.stdout:Total 4.3 MB/s | 53 MB 00:12 2023-09-22T12:46:32.551 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2023-09-22T12:46:32.558 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T12:46:32.579 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T12:46:32.579 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T12:46:32.733 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T12:46:32.891 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T12:46:32.894 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T12:46:33.116 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-09-22T12:46:33.283 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:46:33.370 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T12:46:33.383 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:46:33.613 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-09-22T12:46:33.649 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T12:46:33.825 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:46:33.835 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-09-22T12:46:33.973 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T12:46:34.052 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T12:46:34.246 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : jq-1.6-7.el8.x86_64 4/6 2023-09-22T12:46:34.250 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:46:34.407 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T12:46:34.433 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:46:34.505 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:46:34.713 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 7/25 2023-09-22T12:46:34.999 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T12:46:35.366 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-22T12:46:35.524 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T12:46:35.597 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 9/25 2023-09-22T12:46:35.763 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-09-22T12:46:35.921 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 11/25 2023-09-22T12:46:36.114 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T12:46:36.333 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T12:46:36.541 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 14/25 2023-09-22T12:46:36.724 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T12:46:36.792 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 15/25 2023-09-22T12:46:36.999 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-22T12:46:37.182 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2023-09-22T12:46:37.360 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2023-09-22T12:46:37.533 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 19/25 2023-09-22T12:46:39.028 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T12:46:39.239 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T12:46:39.534 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T12:46:39.534 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 1/6 2023-09-22T12:46:39.534 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 2/6 2023-09-22T12:46:39.534 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : jq-1.6-7.el8.x86_64 3/6 2023-09-22T12:46:39.534 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-09-22T12:46:39.534 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-09-22T12:46:40.002 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:46:40.173 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-09-22T12:46:40.174 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:46:40.174 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T12:46:40.174 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 jq-1.6-7.el8.x86_64 2023-09-22T12:46:40.174 INFO:teuthology.orchestra.run.smithi169.stdout: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 oniguruma-6.8.2-2.el8.x86_64 2023-09-22T12:46:40.174 INFO:teuthology.orchestra.run.smithi169.stdout: socat-1.7.4.1-1.el8.x86_64 xmlstarlet-1.6.1-20.el8.x86_64 2023-09-22T12:46:40.174 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:46:40.175 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T12:46:40.333 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:46:40.369 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph 2023-09-22T12:46:41.023 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 20/25 2023-09-22T12:46:41.234 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:14 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T12:46:41.298 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T12:46:41.398 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 21/25 2023-09-22T12:46:41.515 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:46:42.818 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T12:46:42.821 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:46:42.821 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-09-22T12:46:42.821 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:46:42.821 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T12:46:42.822 INFO:teuthology.orchestra.run.smithi169.stdout: ceph x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.4 k 2023-09-22T12:46:42.822 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-09-22T12:46:42.822 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mds x86_64 2:16.2.14-5.g31c126b7.el8 ceph 2.4 M 2023-09-22T12:46:42.822 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr x86_64 2:16.2.14-5.g31c126b7.el8 ceph 1.6 M 2023-09-22T12:46:42.822 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-modules-core noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 244 k 2023-09-22T12:46:42.822 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mon x86_64 2:16.2.14-5.g31c126b7.el8 ceph 5.0 M 2023-09-22T12:46:42.822 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-osd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 18 M 2023-09-22T12:46:42.823 INFO:teuthology.orchestra.run.smithi169.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-22T12:46:42.823 INFO:teuthology.orchestra.run.smithi169.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-22T12:46:42.823 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-09-22T12:46:42.823 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-22T12:46:42.823 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-22T12:46:42.823 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-09-22T12:46:42.823 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-22T12:46:42.823 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-22T12:46:42.824 INFO:teuthology.orchestra.run.smithi169.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-22T12:46:42.824 INFO:teuthology.orchestra.run.smithi169.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-09-22T12:46:42.824 INFO:teuthology.orchestra.run.smithi169.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-09-22T12:46:42.824 INFO:teuthology.orchestra.run.smithi169.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-22T12:46:42.824 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-22T12:46:42.824 INFO:teuthology.orchestra.run.smithi169.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-22T12:46:42.824 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-09-22T12:46:42.824 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-09-22T12:46:42.825 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-09-22T12:46:42.825 INFO:teuthology.orchestra.run.smithi169.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-22T12:46:42.825 INFO:teuthology.orchestra.run.smithi169.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-22T12:46:42.825 INFO:teuthology.orchestra.run.smithi169.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-22T12:46:42.825 INFO:teuthology.orchestra.run.smithi169.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-22T12:46:42.825 INFO:teuthology.orchestra.run.smithi169.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-22T12:46:42.825 INFO:teuthology.orchestra.run.smithi169.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-22T12:46:42.825 INFO:teuthology.orchestra.run.smithi169.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-22T12:46:42.825 INFO:teuthology.orchestra.run.smithi169.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-09-22T12:46:42.826 INFO:teuthology.orchestra.run.smithi169.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-22T12:46:42.826 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:46:42.826 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T12:46:42.826 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:46:42.826 INFO:teuthology.orchestra.run.smithi169.stdout:Install 32 Packages 2023-09-22T12:46:42.826 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:46:42.827 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 31 M 2023-09-22T12:46:42.827 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 101 M 2023-09-22T12:46:42.827 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T12:46:43.389 INFO:teuthology.orchestra.run.smithi169.stdout:(1/32): ceph-16.2.14-5.g31c126b7.el8.x86_64.rpm 47 kB/s | 6.4 kB 00:00 2023-09-22T12:46:43.555 INFO:teuthology.orchestra.run.smithi169.stdout:(2/32): ceph-mgr-16.2.14-5.g31c126b7.el8.x86_64 5.3 MB/s | 1.6 MB 00:00 2023-09-22T12:46:43.606 INFO:teuthology.orchestra.run.smithi169.stdout:(3/32): ceph-mds-16.2.14-5.g31c126b7.el8.x86_64 6.9 MB/s | 2.4 MB 00:00 2023-09-22T12:46:43.689 INFO:teuthology.orchestra.run.smithi169.stdout:(4/32): ceph-mgr-modules-core-16.2.14-5.g31c126 2.9 MB/s | 244 kB 00:00 2023-09-22T12:46:44.457 INFO:teuthology.orchestra.run.smithi169.stdout:(5/32): ceph-osd-16.2.14-5.g31c126b7.el8.x86_64 20 MB/s | 18 MB 00:00 2023-09-22T12:46:44.749 INFO:teuthology.orchestra.run.smithi169.stdout:(6/32): ceph-mon-16.2.14-5.g31c126b7.el8.x86_64 3.7 MB/s | 5.0 MB 00:01 2023-09-22T12:46:47.091 INFO:teuthology.orchestra.run.smithi169.stdout:(7/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 44 kB/s | 103 kB 00:02 2023-09-22T12:46:48.299 INFO:teuthology.orchestra.run.smithi169.stdout:(8/32): python3-markupsafe-0.23-19.el8.x86_64.r 10 kB/s | 39 kB 00:03 2023-09-22T12:46:51.998 INFO:teuthology.orchestra.run.smithi169.stdout:(9/32): python3-pytz-2017.2-11.el8.noarch.rpm 12 kB/s | 57 kB 00:04 2023-09-22T12:46:52.641 INFO:teuthology.orchestra.run.smithi169.stdout:(10/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 388 kB/s | 249 kB 00:00 2023-09-22T12:46:52.765 INFO:teuthology.orchestra.run.smithi169.stdout:(11/32): python3-mako-1.0.6-14.el8.noarch.rpm 17 kB/s | 157 kB 00:09 2023-09-22T12:46:53.174 INFO:teuthology.orchestra.run.smithi169.stdout:(12/32): python3-pycparser-2.14-14.el8.noarch.r 267 kB/s | 109 kB 00:00 2023-09-22T12:46:53.400 INFO:teuthology.orchestra.run.smithi169.stdout:(13/32): python3-cryptography-3.2.1-6.el8.x86_6 845 kB/s | 641 kB 00:00 2023-09-22T12:46:53.633 INFO:teuthology.orchestra.run.smithi169.stdout:(14/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 95 kB/s | 44 kB 00:00 2023-09-22T12:46:53.867 INFO:teuthology.orchestra.run.smithi169.stdout:(15/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 397 kB/s | 185 kB 00:00 2023-09-22T12:46:53.942 INFO:teuthology.orchestra.run.smithi169.stdout:(16/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 559 kB/s | 173 kB 00:00 2023-09-22T12:46:54.117 INFO:teuthology.orchestra.run.smithi169.stdout:(17/32): python3-jaraco-6.2-6.el8.noarch.rpm 63 kB/s | 11 kB 00:00 2023-09-22T12:46:54.142 INFO:teuthology.orchestra.run.smithi169.stdout:(18/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.4 MB/s | 384 kB 00:00 2023-09-22T12:46:54.284 INFO:teuthology.orchestra.run.smithi169.stdout:(19/32): python3-jaraco-functools-2.0-4.el8.noa 105 kB/s | 18 kB 00:00 2023-09-22T12:46:54.351 INFO:teuthology.orchestra.run.smithi169.stdout:(20/32): python3-logutils-0.3.5-11.el8.noarch.r 233 kB/s | 49 kB 00:00 2023-09-22T12:46:54.485 INFO:teuthology.orchestra.run.smithi169.stdout:(21/32): python3-more-itertools-7.2.0-3.el8.noa 293 kB/s | 59 kB 00:00 2023-09-22T12:46:54.527 INFO:teuthology.orchestra.run.smithi169.stdout:(22/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.6 MB/s | 283 kB 00:00 2023-09-22T12:46:54.635 INFO:teuthology.orchestra.run.smithi169.stdout:(23/32): python3-portend-2.6-1.el8.noarch.rpm 104 kB/s | 16 kB 00:00 2023-09-22T12:46:54.661 INFO:teuthology.orchestra.run.smithi169.stdout:(24/32): python3-simplegeneric-0.8.1-17.el8.noa 140 kB/s | 19 kB 00:00 2023-09-22T12:46:54.778 INFO:teuthology.orchestra.run.smithi169.stdout:(25/32): python3-singledispatch-3.4.0.3-18.el8. 168 kB/s | 24 kB 00:00 2023-09-22T12:46:54.828 INFO:teuthology.orchestra.run.smithi169.stdout:(26/32): python3-tempora-1.14.1-5.el8.noarch.rp 175 kB/s | 29 kB 00:00 2023-09-22T12:46:54.903 INFO:teuthology.orchestra.run.smithi169.stdout:(27/32): python3-trustme-0.6.0-4.el8.noarch.rpm 219 kB/s | 27 kB 00:00 2023-09-22T12:46:54.987 INFO:teuthology.orchestra.run.smithi169.stdout:(28/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.6 MB/s | 255 kB 00:00 2023-09-22T12:46:55.120 INFO:teuthology.orchestra.run.smithi169.stdout:(29/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.1 MB/s | 251 kB 00:00 2023-09-22T12:46:55.146 INFO:teuthology.orchestra.run.smithi169.stdout:(30/32): python3-webtest-2.0.33-1.el8.noarch.rp 538 kB/s | 85 kB 00:00 2023-09-22T12:46:55.262 INFO:teuthology.orchestra.run.smithi169.stdout:(31/32): python3-zc-lockfile-2.0-2.el8.noarch.r 162 kB/s | 23 kB 00:00 2023-09-22T12:46:59.421 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:46:59.422 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /sys 2023-09-22T12:46:59.422 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /proc 2023-09-22T12:46:59.422 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /dev 2023-09-22T12:46:59.422 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /run 2023-09-22T12:46:59.422 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /mnt 2023-09-22T12:46:59.423 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /var/tmp 2023-09-22T12:46:59.423 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /home 2023-09-22T12:46:59.423 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /tmp 2023-09-22T12:46:59.423 INFO:teuthology.orchestra.run.smithi098.stdout:skipping the directory /dev 2023-09-22T12:46:59.423 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:47:00.562 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T12:47:00.643 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 23/25 2023-09-22T12:47:00.643 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T12:47:00.840 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-22T12:47:00.840 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:47:01.541 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:47:01.542 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 1/25 2023-09-22T12:47:01.542 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2/25 2023-09-22T12:47:01.542 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 3/25 2023-09-22T12:47:01.542 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 4/25 2023-09-22T12:47:01.542 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 5/25 2023-09-22T12:47:01.542 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 6/25 2023-09-22T12:47:01.542 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 7/25 2023-09-22T12:47:01.543 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x8 8/25 2023-09-22T12:47:01.543 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_ 9/25 2023-09-22T12:47:01.543 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 10/25 2023-09-22T12:47:01.543 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 11/25 2023-09-22T12:47:01.543 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 12/25 2023-09-22T12:47:01.543 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 13/25 2023-09-22T12:47:01.543 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2023-09-22T12:47:01.543 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2023-09-22T12:47:01.544 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-22T12:47:01.544 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 17/25 2023-09-22T12:47:01.544 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-22T12:47:01.544 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : leveldb-1.22-1.el8.x86_64 19/25 2023-09-22T12:47:01.544 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 20/25 2023-09-22T12:47:01.544 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 21/25 2023-09-22T12:47:01.544 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librados2-2:16.2.14-5.g31c126b7.el8.x86_64 22/25 2023-09-22T12:47:01.544 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-22T12:47:01.545 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 24/25 2023-09-22T12:47:02.574 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-22T12:47:02.574 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:47:02.574 INFO:teuthology.orchestra.run.smithi098.stdout:Upgraded: 2023-09-22T12:47:02.574 INFO:teuthology.orchestra.run.smithi098.stdout: librados2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.574 INFO:teuthology.orchestra.run.smithi098.stdout: librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.574 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T12:47:02.574 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.574 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.575 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-radosgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.575 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-selinux-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.575 INFO:teuthology.orchestra.run.smithi098.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-22T12:47:02.575 INFO:teuthology.orchestra.run.smithi098.stdout: leveldb-1.22-1.el8.x86_64 2023-09-22T12:47:02.575 INFO:teuthology.orchestra.run.smithi098.stdout: libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.575 INFO:teuthology.orchestra.run.smithi098.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-22T12:47:02.575 INFO:teuthology.orchestra.run.smithi098.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-09-22T12:47:02.576 INFO:teuthology.orchestra.run.smithi098.stdout: libradosstriper1-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.576 INFO:teuthology.orchestra.run.smithi098.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-22T12:47:02.576 INFO:teuthology.orchestra.run.smithi098.stdout: librgw2-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.576 INFO:teuthology.orchestra.run.smithi098.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-22T12:47:02.576 INFO:teuthology.orchestra.run.smithi098.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-22T12:47:02.576 INFO:teuthology.orchestra.run.smithi098.stdout: python3-ceph-argparse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.576 INFO:teuthology.orchestra.run.smithi098.stdout: python3-ceph-common-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.577 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.577 INFO:teuthology.orchestra.run.smithi098.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-22T12:47:02.577 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.577 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.577 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:02.577 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:47:02.577 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T12:47:02.851 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph-test 2023-09-22T12:47:03.688 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:00:56 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T12:47:05.250 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T12:47:05.252 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:47:05.252 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repository Size 2023-09-22T12:47:05.252 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:47:05.252 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T12:47:05.252 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-test x86_64 2:16.2.14-5.g31c126b7.el8 ceph 52 M 2023-09-22T12:47:05.252 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2023-09-22T12:47:05.252 INFO:teuthology.orchestra.run.smithi098.stdout: jq x86_64 1.6-7.el8 appstream 202 k 2023-09-22T12:47:05.252 INFO:teuthology.orchestra.run.smithi098.stdout: libcephsqlite x86_64 2:16.2.14-5.g31c126b7.el8 ceph 174 k 2023-09-22T12:47:05.253 INFO:teuthology.orchestra.run.smithi098.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-09-22T12:47:05.253 INFO:teuthology.orchestra.run.smithi098.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-09-22T12:47:05.253 INFO:teuthology.orchestra.run.smithi098.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-09-22T12:47:05.253 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:47:05.253 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T12:47:05.253 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:47:05.253 INFO:teuthology.orchestra.run.smithi098.stdout:Install 6 Packages 2023-09-22T12:47:05.253 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:47:05.254 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 53 M 2023-09-22T12:47:05.254 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 196 M 2023-09-22T12:47:05.254 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T12:47:05.793 INFO:teuthology.orchestra.run.smithi098.stdout:(1/6): libcephsqlite-16.2.14-5.g31c126b7.el8.x8 502 kB/s | 174 kB 00:00 2023-09-22T12:47:07.894 INFO:teuthology.orchestra.run.smithi098.stdout:(2/6): ceph-test-16.2.14-5.g31c126b7.el8.x86_64 21 MB/s | 52 MB 00:02 2023-09-22T12:47:09.495 INFO:teuthology.orchestra.run.smithi169.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 22 kB/s | 457 kB 00:21 2023-09-22T12:47:09.496 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:47:09.497 INFO:teuthology.orchestra.run.smithi169.stdout:Total 1.2 MB/s | 31 MB 00:26 2023-09-22T12:47:09.579 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T12:47:09.618 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T12:47:09.618 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T12:47:09.827 INFO:teuthology.orchestra.run.smithi098.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 46 kB/s | 187 kB 00:04 2023-09-22T12:47:09.940 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T12:47:09.943 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T12:47:11.018 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T12:47:12.297 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-22T12:47:12.493 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-22T12:47:12.691 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-09-22T12:47:12.941 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-09-22T12:47:13.130 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-09-22T12:47:13.287 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-09-22T12:47:13.454 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-09-22T12:47:13.665 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-09-22T12:47:13.865 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-09-22T12:47:14.094 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-09-22T12:47:14.210 INFO:teuthology.orchestra.run.smithi098.stdout:(4/6): jq-1.6-7.el8.x86_64.rpm 23 kB/s | 202 kB 00:08 2023-09-22T12:47:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-09-22T12:47:14.532 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-09-22T12:47:14.594 INFO:teuthology.orchestra.run.smithi098.stdout:(5/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 15 kB/s | 71 kB 00:04 2023-09-22T12:47:14.897 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-09-22T12:47:15.136 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-09-22T12:47:15.348 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-09-22T12:47:15.574 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-09-22T12:47:15.776 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T12:47:16.068 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-09-22T12:47:16.165 INFO:teuthology.orchestra.run.smithi098.stdout:(6/6): socat-1.7.4.1-1.el8.x86_64.rpm 39 kB/s | 323 kB 00:08 2023-09-22T12:47:16.166 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:47:16.166 INFO:teuthology.orchestra.run.smithi098.stdout:Total 4.8 MB/s | 53 MB 00:10 2023-09-22T12:47:16.166 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T12:47:16.183 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T12:47:16.423 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T12:47:16.423 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-09-22T12:47:16.469 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-09-22T12:47:16.699 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-22T12:47:16.700 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T12:47:16.700 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T12:47:16.907 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-22T12:47:17.018 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T12:47:17.118 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-22T12:47:17.231 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-09-22T12:47:17.336 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-09-22T12:47:17.566 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-09-22T12:47:17.566 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T12:47:17.617 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-09-22T12:47:17.791 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : jq-1.6-7.el8.x86_64 4/6 2023-09-22T12:47:17.938 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-09-22T12:47:18.025 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T12:47:18.229 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-22T12:47:18.571 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 27/32 2023-09-22T12:47:18.762 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T12:47:20.243 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T12:47:20.416 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T12:47:20.852 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T12:47:21.049 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T12:47:22.154 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T12:47:22.154 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T12:47:22.154 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T12:47:22.154 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T12:47:22.155 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T12:47:22.155 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 1/32 2023-09-22T12:47:22.155 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2/32 2023-09-22T12:47:22.155 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 3/32 2023-09-22T12:47:22.155 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 4/32 2023-09-22T12:47:22.155 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 5/32 2023-09-22T12:47:22.155 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 6/32 2023-09-22T12:47:22.155 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-09-22T12:47:22.155 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-09-22T12:47:22.155 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-09-22T12:47:22.155 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-09-22T12:47:22.156 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-09-22T12:47:22.157 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-09-22T12:47:22.157 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-09-22T12:47:22.157 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-09-22T12:47:22.157 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-09-22T12:47:22.157 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-09-22T12:47:22.157 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-09-22T12:47:22.157 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-09-22T12:47:22.157 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-09-22T12:47:22.536 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 5/6 2023-09-22T12:47:23.116 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-09-22T12:47:23.116 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:23.116 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T12:47:23.116 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-22T12:47:23.117 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-mako-1.0.6-14.el8.noarch 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-portend-2.6-1.el8.noarch 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pytz-2017.2-11.el8.noarch 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-22T12:47:23.118 INFO:teuthology.orchestra.run.smithi169.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-22T12:47:23.119 INFO:teuthology.orchestra.run.smithi169.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-22T12:47:23.119 INFO:teuthology.orchestra.run.smithi169.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-22T12:47:23.119 INFO:teuthology.orchestra.run.smithi169.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-22T12:47:23.119 INFO:teuthology.orchestra.run.smithi169.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-22T12:47:23.119 INFO:teuthology.orchestra.run.smithi169.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-22T12:47:23.119 INFO:teuthology.orchestra.run.smithi169.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-22T12:47:23.119 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:23.119 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T12:47:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T12:47:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 6/6 2023-09-22T12:47:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 1/6 2023-09-22T12:47:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 2/6 2023-09-22T12:47:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : jq-1.6-7.el8.x86_64 3/6 2023-09-22T12:47:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-09-22T12:47:23.120 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-09-22T12:47:23.210 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-base 2023-09-22T12:47:23.659 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-09-22T12:47:23.773 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:47:23.773 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T12:47:23.773 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-test-2:16.2.14-5.g31c126b7.el8.x86_64 jq-1.6-7.el8.x86_64 2023-09-22T12:47:23.773 INFO:teuthology.orchestra.run.smithi098.stdout: libcephsqlite-2:16.2.14-5.g31c126b7.el8.x86_64 oniguruma-6.8.2-2.el8.x86_64 2023-09-22T12:47:23.773 INFO:teuthology.orchestra.run.smithi098.stdout: socat-1.7.4.1-1.el8.x86_64 xmlstarlet-1.6.1-20.el8.x86_64 2023-09-22T12:47:23.773 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:47:23.774 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T12:47:23.870 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph 2023-09-22T12:47:24.067 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:57 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T12:47:24.726 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:01:17 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T12:47:25.463 INFO:teuthology.orchestra.run.smithi169.stdout:Package ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T12:47:25.642 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T12:47:25.644 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-09-22T12:47:25.644 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T12:47:25.745 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install cephadm 2023-09-22T12:47:26.271 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T12:47:26.274 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:47:26.275 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repository Size 2023-09-22T12:47:26.275 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:47:26.275 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T12:47:26.275 INFO:teuthology.orchestra.run.smithi098.stdout: ceph x86_64 2:16.2.14-5.g31c126b7.el8 ceph 6.4 k 2023-09-22T12:47:26.275 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2023-09-22T12:47:26.275 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mds x86_64 2:16.2.14-5.g31c126b7.el8 ceph 2.4 M 2023-09-22T12:47:26.275 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr x86_64 2:16.2.14-5.g31c126b7.el8 ceph 1.6 M 2023-09-22T12:47:26.275 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-modules-core noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 244 k 2023-09-22T12:47:26.276 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mon x86_64 2:16.2.14-5.g31c126b7.el8 ceph 5.0 M 2023-09-22T12:47:26.276 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-osd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 18 M 2023-09-22T12:47:26.276 INFO:teuthology.orchestra.run.smithi098.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-22T12:47:26.276 INFO:teuthology.orchestra.run.smithi098.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-22T12:47:26.276 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-09-22T12:47:26.276 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-22T12:47:26.276 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-22T12:47:26.276 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-09-22T12:47:26.277 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-22T12:47:26.277 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-22T12:47:26.277 INFO:teuthology.orchestra.run.smithi098.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-22T12:47:26.277 INFO:teuthology.orchestra.run.smithi098.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-09-22T12:47:26.277 INFO:teuthology.orchestra.run.smithi098.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-09-22T12:47:26.277 INFO:teuthology.orchestra.run.smithi098.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-22T12:47:26.277 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-22T12:47:26.277 INFO:teuthology.orchestra.run.smithi098.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-22T12:47:26.277 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-09-22T12:47:26.278 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-09-22T12:47:26.278 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-09-22T12:47:26.278 INFO:teuthology.orchestra.run.smithi098.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-22T12:47:26.278 INFO:teuthology.orchestra.run.smithi098.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-22T12:47:26.278 INFO:teuthology.orchestra.run.smithi098.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-22T12:47:26.278 INFO:teuthology.orchestra.run.smithi098.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-22T12:47:26.278 INFO:teuthology.orchestra.run.smithi098.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-22T12:47:26.278 INFO:teuthology.orchestra.run.smithi098.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-22T12:47:26.279 INFO:teuthology.orchestra.run.smithi098.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-22T12:47:26.279 INFO:teuthology.orchestra.run.smithi098.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-09-22T12:47:26.279 INFO:teuthology.orchestra.run.smithi098.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-22T12:47:26.279 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:47:26.279 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T12:47:26.279 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:47:26.279 INFO:teuthology.orchestra.run.smithi098.stdout:Install 32 Packages 2023-09-22T12:47:26.279 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:47:26.280 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 31 M 2023-09-22T12:47:26.280 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 101 M 2023-09-22T12:47:26.280 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T12:47:26.595 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:59 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T12:47:26.921 INFO:teuthology.orchestra.run.smithi098.stdout:(1/32): ceph-16.2.14-5.g31c126b7.el8.x86_64.rpm 31 kB/s | 6.4 kB 00:00 2023-09-22T12:47:26.980 INFO:teuthology.orchestra.run.smithi098.stdout:(2/32): ceph-mgr-16.2.14-5.g31c126b7.el8.x86_64 6.1 MB/s | 1.6 MB 00:00 2023-09-22T12:47:27.863 INFO:teuthology.orchestra.run.smithi098.stdout:(3/32): ceph-osd-16.2.14-5.g31c126b7.el8.x86_64 21 MB/s | 18 MB 00:00 2023-09-22T12:47:27.914 INFO:teuthology.orchestra.run.smithi098.stdout:(4/32): ceph-mgr-modules-core-16.2.14-5.g31c126 4.8 MB/s | 244 kB 00:00 2023-09-22T12:47:28.160 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T12:47:28.161 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:47:28.161 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-09-22T12:47:28.161 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:47:28.161 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T12:47:28.162 INFO:teuthology.orchestra.run.smithi169.stdout: cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 84 k 2023-09-22T12:47:28.162 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:28.162 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T12:47:28.162 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:47:28.162 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-09-22T12:47:28.162 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:28.162 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 84 k 2023-09-22T12:47:28.162 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 348 k 2023-09-22T12:47:28.163 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T12:47:28.289 INFO:teuthology.orchestra.run.smithi098.stdout:(5/32): ceph-mds-16.2.14-5.g31c126b7.el8.x86_64 1.6 MB/s | 2.4 MB 00:01 2023-09-22T12:47:28.425 INFO:teuthology.orchestra.run.smithi169.stdout:cephadm-16.2.14-5.g31c126b7.el8.noarch.rpm 321 kB/s | 84 kB 00:00 2023-09-22T12:47:28.425 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:47:28.425 INFO:teuthology.orchestra.run.smithi169.stdout:Total 319 kB/s | 84 kB 00:00 2023-09-22T12:47:28.426 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T12:47:28.432 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T12:47:28.432 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T12:47:28.434 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T12:47:28.437 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T12:47:28.480 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T12:47:28.789 INFO:teuthology.orchestra.run.smithi098.stdout:(6/32): ceph-mon-16.2.14-5.g31c126b7.el8.x86_64 2.7 MB/s | 5.0 MB 00:01 2023-09-22T12:47:28.958 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:47:29.106 INFO:teuthology.orchestra.run.smithi098.stdout:(7/32): python3-markupsafe-0.23-19.el8.x86_64.r 47 kB/s | 39 kB 00:00 2023-09-22T12:47:29.114 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:47:29.342 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:47:29.957 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:47:29.957 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:29.957 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T12:47:29.957 INFO:teuthology.orchestra.run.smithi169.stdout: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:47:29.957 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:29.957 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T12:47:30.173 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-immutable-object-cache 2023-09-22T12:47:31.028 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:04 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T12:47:32.531 INFO:teuthology.orchestra.run.smithi098.stdout:(8/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 28 kB/s | 103 kB 00:03 2023-09-22T12:47:32.605 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T12:47:32.606 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:47:32.606 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repo Size 2023-09-22T12:47:32.606 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:47:32.606 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T12:47:32.606 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-5.g31c126b7.el8 ceph 161 k 2023-09-22T12:47:32.606 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:32.606 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T12:47:32.607 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:47:32.607 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-09-22T12:47:32.607 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:32.607 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 161 k 2023-09-22T12:47:32.607 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 470 k 2023-09-22T12:47:32.607 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T12:47:32.789 INFO:teuthology.orchestra.run.smithi169.stdout:ceph-immutable-object-cache-16.2.14-5.g31c126b7 885 kB/s | 161 kB 00:00 2023-09-22T12:47:32.790 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:47:32.790 INFO:teuthology.orchestra.run.smithi169.stdout:Total 879 kB/s | 161 kB 00:00 2023-09-22T12:47:32.791 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T12:47:32.801 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T12:47:32.801 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T12:47:32.848 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T12:47:32.851 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T12:47:32.960 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T12:47:33.122 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:47:33.554 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:47:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:47:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:34.152 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T12:47:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:34.153 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T12:47:34.372 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-mgr 2023-09-22T12:47:35.232 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:08 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T12:47:35.973 INFO:teuthology.orchestra.run.smithi098.stdout:(9/32): python3-mako-1.0.6-14.el8.noarch.rpm 19 kB/s | 157 kB 00:08 2023-09-22T12:47:36.398 INFO:teuthology.orchestra.run.smithi098.stdout:(10/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 586 kB/s | 249 kB 00:00 2023-09-22T12:47:36.565 INFO:teuthology.orchestra.run.smithi098.stdout:(11/32): python3-cryptography-3.2.1-6.el8.x86_6 3.8 MB/s | 641 kB 00:00 2023-09-22T12:47:36.627 INFO:teuthology.orchestra.run.smithi169.stdout:Package ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T12:47:36.657 INFO:teuthology.orchestra.run.smithi098.stdout:(12/32): python3-pycparser-2.14-14.el8.noarch.r 1.2 MB/s | 109 kB 00:00 2023-09-22T12:47:36.808 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T12:47:36.810 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-09-22T12:47:36.810 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T12:47:36.910 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-mgr-dashboard 2023-09-22T12:47:36.916 INFO:teuthology.orchestra.run.smithi098.stdout:(13/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 169 kB/s | 44 kB 00:00 2023-09-22T12:47:37.066 INFO:teuthology.orchestra.run.smithi098.stdout:(14/32): python3-pytz-2017.2-11.el8.noarch.rpm 7.2 kB/s | 57 kB 00:07 2023-09-22T12:47:37.100 INFO:teuthology.orchestra.run.smithi098.stdout:(15/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 1.0 MB/s | 185 kB 00:00 2023-09-22T12:47:37.250 INFO:teuthology.orchestra.run.smithi098.stdout:(16/32): python3-cherrypy-18.4.0-1.el8.noarch.r 2.5 MB/s | 384 kB 00:00 2023-09-22T12:47:37.325 INFO:teuthology.orchestra.run.smithi098.stdout:(17/32): python3-jaraco-6.2-6.el8.noarch.rpm 148 kB/s | 11 kB 00:00 2023-09-22T12:47:37.400 INFO:teuthology.orchestra.run.smithi098.stdout:(18/32): python3-jaraco-functools-2.0-4.el8.noa 235 kB/s | 18 kB 00:00 2023-09-22T12:47:37.425 INFO:teuthology.orchestra.run.smithi098.stdout:(19/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 480 kB/s | 173 kB 00:00 2023-09-22T12:47:37.476 INFO:teuthology.orchestra.run.smithi098.stdout:(20/32): python3-logutils-0.3.5-11.el8.noarch.r 648 kB/s | 49 kB 00:00 2023-09-22T12:47:37.510 INFO:teuthology.orchestra.run.smithi098.stdout:(21/32): python3-more-itertools-7.2.0-3.el8.noa 702 kB/s | 59 kB 00:00 2023-09-22T12:47:37.568 INFO:teuthology.orchestra.run.smithi098.stdout:(22/32): python3-pecan-1.3.2-9.el8.noarch.rpm 3.0 MB/s | 283 kB 00:00 2023-09-22T12:47:37.593 INFO:teuthology.orchestra.run.smithi098.stdout:(23/32): python3-portend-2.6-1.el8.noarch.rpm 187 kB/s | 16 kB 00:00 2023-09-22T12:47:37.644 INFO:teuthology.orchestra.run.smithi098.stdout:(24/32): python3-simplegeneric-0.8.1-17.el8.noa 249 kB/s | 19 kB 00:00 2023-09-22T12:47:37.669 INFO:teuthology.orchestra.run.smithi098.stdout:(25/32): python3-singledispatch-3.4.0.3-18.el8. 317 kB/s | 24 kB 00:00 2023-09-22T12:47:37.720 INFO:teuthology.orchestra.run.smithi098.stdout:(26/32): python3-tempora-1.14.1-5.el8.noarch.rp 388 kB/s | 29 kB 00:00 2023-09-22T12:47:37.745 INFO:teuthology.orchestra.run.smithi098.stdout:(27/32): python3-trustme-0.6.0-4.el8.noarch.rpm 365 kB/s | 27 kB 00:00 2023-09-22T12:47:37.767 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:10 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T12:47:37.812 INFO:teuthology.orchestra.run.smithi098.stdout:(28/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.7 MB/s | 255 kB 00:00 2023-09-22T12:47:37.870 INFO:teuthology.orchestra.run.smithi098.stdout:(29/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.0 MB/s | 251 kB 00:00 2023-09-22T12:47:37.895 INFO:teuthology.orchestra.run.smithi098.stdout:(30/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.0 MB/s | 85 kB 00:00 2023-09-22T12:47:37.938 INFO:teuthology.orchestra.run.smithi098.stdout:(31/32): python3-zc-lockfile-2.0-2.el8.noarch.r 343 kB/s | 23 kB 00:00 2023-09-22T12:47:39.353 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T12:47:39.354 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:47:39.355 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-09-22T12:47:39.355 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:47:39.355 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T12:47:39.355 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-dashboard noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 3.1 M 2023-09-22T12:47:39.355 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-09-22T12:47:39.355 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-grafana-dashboards noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 24 k 2023-09-22T12:47:39.355 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-prometheus-alerts noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 15 k 2023-09-22T12:47:39.355 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-09-22T12:47:39.355 INFO:teuthology.orchestra.run.smithi169.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-22T12:47:39.356 INFO:teuthology.orchestra.run.smithi169.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-22T12:47:39.356 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:39.356 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T12:47:39.356 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T12:47:39.356 INFO:teuthology.orchestra.run.smithi169.stdout:Install 6 Packages 2023-09-22T12:47:39.356 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:39.357 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 3.4 M 2023-09-22T12:47:39.357 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 22 M 2023-09-22T12:47:39.357 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T12:47:39.617 INFO:teuthology.orchestra.run.smithi169.stdout:(1/6): ceph-grafana-dashboards-16.2.14-5.g31c12 152 kB/s | 24 kB 00:00 2023-09-22T12:47:39.717 INFO:teuthology.orchestra.run.smithi169.stdout:(2/6): ceph-prometheus-alerts-16.2.14-5.g31c126 57 kB/s | 15 kB 00:00 2023-09-22T12:47:39.901 INFO:teuthology.orchestra.run.smithi169.stdout:(3/6): python3-jwt-1.6.1-2.el8.noarch.rpm 150 kB/s | 43 kB 00:00 2023-09-22T12:47:39.951 INFO:teuthology.orchestra.run.smithi169.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 144 kB/s | 34 kB 00:00 2023-09-22T12:47:40.018 INFO:teuthology.orchestra.run.smithi169.stdout:(5/6): ceph-mgr-dashboard-16.2.14-5.g31c126b7.e 5.6 MB/s | 3.1 MB 00:00 2023-09-22T12:47:40.252 INFO:teuthology.orchestra.run.smithi169.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 560 kB/s | 196 kB 00:00 2023-09-22T12:47:40.255 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:47:40.255 INFO:teuthology.orchestra.run.smithi169.stdout:Total 3.8 MB/s | 3.4 MB 00:00 2023-09-22T12:47:40.266 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T12:47:40.276 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T12:47:40.276 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T12:47:40.439 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T12:47:40.449 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T12:47:40.704 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T12:47:40.877 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-09-22T12:47:41.072 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-09-22T12:47:41.232 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-09-22T12:47:41.335 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 4/6 2023-09-22T12:47:42.343 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 5/6 2023-09-22T12:47:42.566 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T12:47:42.859 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T12:47:42.859 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 1/6 2023-09-22T12:47:42.860 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2/6 2023-09-22T12:47:42.860 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 3/6 2023-09-22T12:47:42.860 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-09-22T12:47:42.860 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-09-22T12:47:43.476 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-09-22T12:47:43.476 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:43.477 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T12:47:43.477 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:47:43.477 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:47:43.477 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:47:43.477 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-22T12:47:43.477 INFO:teuthology.orchestra.run.smithi169.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-22T12:47:43.478 INFO:teuthology.orchestra.run.smithi169.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-22T12:47:43.478 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:43.478 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T12:47:43.591 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-22T12:47:44.287 INFO:teuthology.orchestra.run.smithi098.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 39 kB/s | 457 kB 00:11 2023-09-22T12:47:44.290 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:47:44.290 INFO:teuthology.orchestra.run.smithi098.stdout:Total 1.7 MB/s | 31 MB 00:18 2023-09-22T12:47:44.376 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T12:47:44.415 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T12:47:44.415 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T12:47:44.449 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:17 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T12:47:44.731 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T12:47:44.734 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T12:47:45.801 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T12:47:46.012 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-22T12:47:46.034 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T12:47:46.036 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================================ 2023-09-22T12:47:46.036 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-09-22T12:47:46.036 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================================ 2023-09-22T12:47:46.036 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T12:47:46.036 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 8.6 M 2023-09-22T12:47:46.036 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-09-22T12:47:46.036 INFO:teuthology.orchestra.run.smithi169.stdout: libgfortran x86_64 8.5.0-20.el8 baseos 645 k 2023-09-22T12:47:46.036 INFO:teuthology.orchestra.run.smithi169.stdout: libquadmath x86_64 8.5.0-20.el8 baseos 172 k 2023-09-22T12:47:46.037 INFO:teuthology.orchestra.run.smithi169.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-09-22T12:47:46.037 INFO:teuthology.orchestra.run.smithi169.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-09-22T12:47:46.037 INFO:teuthology.orchestra.run.smithi169.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-09-22T12:47:46.037 INFO:teuthology.orchestra.run.smithi169.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-09-22T12:47:46.037 INFO:teuthology.orchestra.run.smithi169.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-09-22T12:47:46.037 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:46.037 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T12:47:46.037 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================================ 2023-09-22T12:47:46.037 INFO:teuthology.orchestra.run.smithi169.stdout:Install 8 Packages 2023-09-22T12:47:46.037 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T12:47:46.038 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 37 M 2023-09-22T12:47:46.038 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 210 M 2023-09-22T12:47:46.038 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T12:47:46.214 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-22T12:47:46.423 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-09-22T12:47:46.696 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-09-22T12:47:46.894 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-09-22T12:47:47.091 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-09-22T12:47:47.268 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-09-22T12:47:47.462 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-09-22T12:47:47.655 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-09-22T12:47:47.763 INFO:teuthology.orchestra.run.smithi169.stdout:(1/8): ceph-mgr-diskprediction-local-16.2.14-5. 5.7 MB/s | 8.6 MB 00:01 2023-09-22T12:47:47.876 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-09-22T12:47:48.072 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-09-22T12:47:48.304 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-09-22T12:47:48.694 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-09-22T12:47:48.967 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-09-22T12:47:49.199 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-09-22T12:47:49.421 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-09-22T12:47:49.632 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T12:47:49.932 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-09-22T12:47:50.162 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-09-22T12:47:51.982 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-09-22T12:47:52.570 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-22T12:47:52.844 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-22T12:47:53.086 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-22T12:47:53.311 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-09-22T12:47:53.624 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-09-22T12:47:53.934 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-22T12:47:54.281 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 27/32 2023-09-22T12:47:54.447 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T12:47:55.938 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 28/32 2023-09-22T12:47:56.106 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T12:47:56.535 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 29/32 2023-09-22T12:47:56.722 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T12:47:56.939 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 30/32 2023-09-22T12:47:57.133 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T12:47:57.151 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 31/32 2023-09-22T12:47:57.263 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T12:47:57.670 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 32/32 2023-09-22T12:47:57.671 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-2:16.2.14-5.g31c126b7.el8.x86_64 1/32 2023-09-22T12:47:57.671 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2/32 2023-09-22T12:47:57.671 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 3/32 2023-09-22T12:47:57.671 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 4/32 2023-09-22T12:47:57.671 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 5/32 2023-09-22T12:47:57.671 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.no 6/32 2023-09-22T12:47:57.672 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-09-22T12:47:57.672 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-09-22T12:47:57.672 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-09-22T12:47:57.672 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-09-22T12:47:57.672 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-09-22T12:47:57.672 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-09-22T12:47:57.672 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-09-22T12:47:57.673 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-09-22T12:47:57.673 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-09-22T12:47:57.673 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-09-22T12:47:57.673 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-09-22T12:47:57.673 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-09-22T12:47:57.673 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-09-22T12:47:57.673 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-09-22T12:47:57.673 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-09-22T12:47:57.674 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-09-22T12:47:57.674 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-09-22T12:47:57.674 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-09-22T12:47:57.674 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-09-22T12:47:57.674 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-09-22T12:47:57.674 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-09-22T12:47:57.674 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-09-22T12:47:57.674 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-09-22T12:47:57.675 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-09-22T12:47:57.675 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-09-22T12:47:58.720 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-09-22T12:47:58.721 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:47:58.721 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T12:47:58.721 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:58.721 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mds-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:58.721 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:58.721 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-modules-core-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:47:58.722 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mon-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:58.722 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-osd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:47:58.722 INFO:teuthology.orchestra.run.smithi098.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-22T12:47:58.722 INFO:teuthology.orchestra.run.smithi098.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-22T12:47:58.722 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-09-22T12:47:58.722 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-22T12:47:58.722 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-22T12:47:58.722 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-09-22T12:47:58.723 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-22T12:47:58.723 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-22T12:47:58.723 INFO:teuthology.orchestra.run.smithi098.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-22T12:47:58.723 INFO:teuthology.orchestra.run.smithi098.stdout: python3-mako-1.0.6-14.el8.noarch 2023-09-22T12:47:58.723 INFO:teuthology.orchestra.run.smithi098.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-22T12:47:58.723 INFO:teuthology.orchestra.run.smithi098.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-22T12:47:58.723 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-22T12:47:58.723 INFO:teuthology.orchestra.run.smithi098.stdout: python3-portend-2.6-1.el8.noarch 2023-09-22T12:47:58.724 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-22T12:47:58.724 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-22T12:47:58.724 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pytz-2017.2-11.el8.noarch 2023-09-22T12:47:58.724 INFO:teuthology.orchestra.run.smithi098.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-22T12:47:58.724 INFO:teuthology.orchestra.run.smithi098.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-22T12:47:58.724 INFO:teuthology.orchestra.run.smithi098.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-22T12:47:58.724 INFO:teuthology.orchestra.run.smithi098.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-22T12:47:58.724 INFO:teuthology.orchestra.run.smithi098.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-22T12:47:58.725 INFO:teuthology.orchestra.run.smithi098.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-22T12:47:58.725 INFO:teuthology.orchestra.run.smithi098.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-22T12:47:58.725 INFO:teuthology.orchestra.run.smithi098.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-22T12:47:58.725 INFO:teuthology.orchestra.run.smithi098.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-22T12:47:58.725 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:47:58.725 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T12:47:58.932 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph-base 2023-09-22T12:47:59.779 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:01:52 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T12:48:01.165 INFO:teuthology.orchestra.run.smithi098.stdout:Package ceph-base-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T12:48:01.345 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T12:48:01.347 INFO:teuthology.orchestra.run.smithi098.stdout:Nothing to do. 2023-09-22T12:48:01.347 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T12:48:01.438 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install cephadm 2023-09-22T12:48:02.285 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:01:55 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T12:48:03.853 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T12:48:03.854 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:48:03.854 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repository Size 2023-09-22T12:48:03.854 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:48:03.854 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T12:48:03.854 INFO:teuthology.orchestra.run.smithi098.stdout: cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 84 k 2023-09-22T12:48:03.854 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:03.854 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T12:48:03.854 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:48:03.854 INFO:teuthology.orchestra.run.smithi098.stdout:Install 1 Package 2023-09-22T12:48:03.855 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:03.855 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 84 k 2023-09-22T12:48:03.855 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 348 k 2023-09-22T12:48:03.855 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T12:48:04.011 INFO:teuthology.orchestra.run.smithi098.stdout:cephadm-16.2.14-5.g31c126b7.el8.noarch.rpm 537 kB/s | 84 kB 00:00 2023-09-22T12:48:04.012 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:48:04.012 INFO:teuthology.orchestra.run.smithi098.stdout:Total 533 kB/s | 84 kB 00:00 2023-09-22T12:48:04.012 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T12:48:04.019 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T12:48:04.019 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T12:48:04.021 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T12:48:04.024 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T12:48:04.064 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T12:48:04.555 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:48:04.724 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:48:04.949 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:48:05.556 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/1 2023-09-22T12:48:05.556 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:05.556 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T12:48:05.556 INFO:teuthology.orchestra.run.smithi098.stdout: cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:48:05.557 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:05.557 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T12:48:05.757 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph-immutable-object-cache 2023-09-22T12:48:06.609 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:01:59 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T12:48:08.150 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T12:48:08.150 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:48:08.151 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repo Size 2023-09-22T12:48:08.151 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:48:08.151 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T12:48:08.151 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-5.g31c126b7.el8 ceph 161 k 2023-09-22T12:48:08.151 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:08.151 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T12:48:08.151 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:48:08.151 INFO:teuthology.orchestra.run.smithi098.stdout:Install 1 Package 2023-09-22T12:48:08.151 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:08.152 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 161 k 2023-09-22T12:48:08.152 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 470 k 2023-09-22T12:48:08.152 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T12:48:08.340 INFO:teuthology.orchestra.run.smithi098.stdout:ceph-immutable-object-cache-16.2.14-5.g31c126b7 855 kB/s | 161 kB 00:00 2023-09-22T12:48:08.341 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:48:08.341 INFO:teuthology.orchestra.run.smithi098.stdout:Total 849 kB/s | 161 kB 00:00 2023-09-22T12:48:08.342 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T12:48:08.352 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T12:48:08.352 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T12:48:08.398 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T12:48:08.401 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T12:48:08.510 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T12:48:08.696 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:48:09.116 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:48:09.722 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el 1/1 2023-09-22T12:48:09.722 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:09.722 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T12:48:09.723 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-immutable-object-cache-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T12:48:09.723 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:09.723 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T12:48:09.929 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph-mgr 2023-09-22T12:48:10.783 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:02:03 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T12:48:12.157 INFO:teuthology.orchestra.run.smithi098.stdout:Package ceph-mgr-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T12:48:12.339 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T12:48:12.340 INFO:teuthology.orchestra.run.smithi098.stdout:Nothing to do. 2023-09-22T12:48:12.340 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T12:48:12.430 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph-mgr-dashboard 2023-09-22T12:48:13.284 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:02:06 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T12:48:14.844 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T12:48:14.845 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:48:14.846 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repository Size 2023-09-22T12:48:14.846 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:48:14.846 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T12:48:14.846 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-dashboard noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 3.1 M 2023-09-22T12:48:14.846 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2023-09-22T12:48:14.846 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-grafana-dashboards noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 24 k 2023-09-22T12:48:14.846 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-prometheus-alerts noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 15 k 2023-09-22T12:48:14.846 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-09-22T12:48:14.846 INFO:teuthology.orchestra.run.smithi098.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-22T12:48:14.846 INFO:teuthology.orchestra.run.smithi098.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-22T12:48:14.846 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:14.847 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T12:48:14.847 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T12:48:14.847 INFO:teuthology.orchestra.run.smithi098.stdout:Install 6 Packages 2023-09-22T12:48:14.847 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:14.847 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 3.4 M 2023-09-22T12:48:14.847 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 22 M 2023-09-22T12:48:14.848 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T12:48:15.168 INFO:teuthology.orchestra.run.smithi098.stdout:(1/6): ceph-grafana-dashboards-16.2.14-5.g31c12 106 kB/s | 24 kB 00:00 2023-09-22T12:48:15.193 INFO:teuthology.orchestra.run.smithi098.stdout:(2/6): ceph-prometheus-alerts-16.2.14-5.g31c126 58 kB/s | 15 kB 00:00 2023-09-22T12:48:15.310 INFO:teuthology.orchestra.run.smithi098.stdout:(3/6): ceph-mgr-dashboard-16.2.14-5.g31c126b7.e 8.4 MB/s | 3.1 MB 00:00 2023-09-22T12:48:15.336 INFO:teuthology.orchestra.run.smithi098.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 237 kB/s | 34 kB 00:00 2023-09-22T12:48:15.394 INFO:teuthology.orchestra.run.smithi098.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 2.3 MB/s | 196 kB 00:00 2023-09-22T12:48:15.553 INFO:teuthology.orchestra.run.smithi098.stdout:(6/6): python3-jwt-1.6.1-2.el8.noarch.rpm 111 kB/s | 43 kB 00:00 2023-09-22T12:48:15.555 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T12:48:15.555 INFO:teuthology.orchestra.run.smithi098.stdout:Total 4.8 MB/s | 3.4 MB 00:00 2023-09-22T12:48:15.565 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T12:48:15.576 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T12:48:15.576 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T12:48:15.740 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T12:48:15.743 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T12:48:16.014 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T12:48:16.234 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-09-22T12:48:16.445 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-09-22T12:48:16.630 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-09-22T12:48:16.759 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 4/6 2023-09-22T12:48:17.768 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 5/6 2023-09-22T12:48:18.010 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T12:48:18.303 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T12:48:18.304 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.no 1/6 2023-09-22T12:48:18.304 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2/6 2023-09-22T12:48:18.304 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noa 3/6 2023-09-22T12:48:18.304 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-09-22T12:48:18.304 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-09-22T12:48:19.034 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-09-22T12:48:19.034 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:19.034 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T12:48:19.034 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-grafana-dashboards-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:48:19.035 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-dashboard-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:48:19.035 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-prometheus-alerts-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T12:48:19.035 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-22T12:48:19.035 INFO:teuthology.orchestra.run.smithi098.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-22T12:48:19.035 INFO:teuthology.orchestra.run.smithi098.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-22T12:48:19.036 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:19.036 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T12:48:19.145 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-22T12:48:19.992 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:02:12 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T12:48:21.567 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T12:48:21.568 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================================ 2023-09-22T12:48:21.569 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repository Size 2023-09-22T12:48:21.569 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================================ 2023-09-22T12:48:21.569 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T12:48:21.569 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 8.6 M 2023-09-22T12:48:21.569 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2023-09-22T12:48:21.569 INFO:teuthology.orchestra.run.smithi098.stdout: libgfortran x86_64 8.5.0-20.el8 baseos 645 k 2023-09-22T12:48:21.569 INFO:teuthology.orchestra.run.smithi098.stdout: libquadmath x86_64 8.5.0-20.el8 baseos 172 k 2023-09-22T12:48:21.569 INFO:teuthology.orchestra.run.smithi098.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-09-22T12:48:21.569 INFO:teuthology.orchestra.run.smithi098.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-09-22T12:48:21.569 INFO:teuthology.orchestra.run.smithi098.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-09-22T12:48:21.570 INFO:teuthology.orchestra.run.smithi098.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-09-22T12:48:21.570 INFO:teuthology.orchestra.run.smithi098.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-09-22T12:48:21.570 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:21.570 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T12:48:21.570 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================================ 2023-09-22T12:48:21.570 INFO:teuthology.orchestra.run.smithi098.stdout:Install 8 Packages 2023-09-22T12:48:21.570 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T12:48:21.571 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 37 M 2023-09-22T12:48:21.571 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 210 M 2023-09-22T12:48:21.571 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T12:48:22.574 INFO:teuthology.orchestra.run.smithi098.stdout:(1/8): ceph-mgr-diskprediction-local-16.2.14-5. 12 MB/s | 8.6 MB 00:00 2023-09-22T12:50:57.885 INFO:teuthology.orchestra.run.smithi169.stdout:(2/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 20 kB/s | 3.7 MB 03:10 2023-09-22T12:51:00.636 INFO:teuthology.orchestra.run.smithi098.stdout:(2/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 24 kB/s | 3.7 MB 02:38 2023-09-22T12:51:09.152 INFO:teuthology.orchestra.run.smithi098.stdout:(3/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 26 kB/s | 225 kB 00:08 2023-09-22T12:51:11.999 INFO:teuthology.orchestra.run.smithi169.stdout:(3/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 16 kB/s | 225 kB 00:14 2023-09-22T12:51:25.115 INFO:teuthology.orchestra.run.smithi169.stdout:(4/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 23 kB/s | 4.9 MB 03:38 2023-09-22T12:51:25.598 INFO:teuthology.orchestra.run.smithi169.stdout:(5/8): libgfortran-8.5.0-20.el8.x86_64.rpm 1.3 MB/s | 645 kB 00:00 2023-09-22T12:51:25.690 INFO:teuthology.orchestra.run.smithi169.stdout:(6/8): libquadmath-8.5.0-20.el8.x86_64.rpm 1.8 MB/s | 172 kB 00:00 2023-09-22T12:51:38.657 INFO:teuthology.orchestra.run.smithi098.stdout:(4/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 25 kB/s | 4.9 MB 03:16 2023-09-22T12:51:39.141 INFO:teuthology.orchestra.run.smithi098.stdout:(5/8): libgfortran-8.5.0-20.el8.x86_64.rpm 1.3 MB/s | 645 kB 00:00 2023-09-22T12:51:39.233 INFO:teuthology.orchestra.run.smithi098.stdout:(6/8): libquadmath-8.5.0-20.el8.x86_64.rpm 1.8 MB/s | 172 kB 00:00 2023-09-22T12:52:13.454 INFO:teuthology.orchestra.run.smithi098.stdout:(7/8): openblas-0.3.15-4.el8.x86_64.rpm 21 kB/s | 4.7 MB 03:51 2023-09-22T12:56:03.691 INFO:teuthology.orchestra.run.smithi169.stdout:[MIRROR] openblas-0.3.15-4.el8.x86_64.rpm: Curl error (28): Timeout was reached for http://mirror.ash.fastserv.com/centos/8-stream/AppStream/x86_64/os/Packages/openblas-0.3.15-4.el8.x86_64.rpm [Operation too slow. Less than 1000 bytes/sec transferred the last 300 seconds] 2023-09-22T12:56:04.040 INFO:teuthology.orchestra.run.smithi169.stdout:(7/8): openblas-0.3.15-4.el8.x86_64.rpm 9.7 kB/s | 4.7 MB 08:17 2023-09-22T13:01:09.504 INFO:teuthology.orchestra.run.smithi169.stdout:(8/8): python3-scipy-1.0.0-21.module_el8.5.0+77 23 kB/s | 14 MB 09:57 2023-09-22T13:01:09.504 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:09.505 INFO:teuthology.orchestra.run.smithi169.stdout:Total 47 kB/s | 37 MB 13:23 2023-09-22T13:01:09.732 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T13:01:09.750 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T13:01:09.750 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T13:01:10.087 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T13:01:10.090 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T13:01:10.665 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T13:01:10.822 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T13:01:10.956 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T13:01:11.089 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T13:01:11.667 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T13:01:11.847 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:01:12.450 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:01:12.605 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T13:01:13.311 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T13:01:13.625 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T13:01:13.863 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T13:01:15.628 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T13:01:16.923 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-09-22T13:01:17.388 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T13:01:17.712 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T13:01:17.712 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 1/8 2023-09-22T13:01:17.712 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-09-22T13:01:17.712 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:01:17.712 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-09-22T13:01:17.712 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T13:01:17.712 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-09-22T13:01:17.713 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 7/8 2023-09-22T13:01:18.362 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 8/8 2023-09-22T13:01:18.362 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:01:18.362 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T13:01:18.362 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:01:18.363 INFO:teuthology.orchestra.run.smithi169.stdout: libgfortran-8.5.0-20.el8.x86_64 2023-09-22T13:01:18.363 INFO:teuthology.orchestra.run.smithi169.stdout: libquadmath-8.5.0-20.el8.x86_64 2023-09-22T13:01:18.363 INFO:teuthology.orchestra.run.smithi169.stdout: openblas-0.3.15-4.el8.x86_64 2023-09-22T13:01:18.363 INFO:teuthology.orchestra.run.smithi169.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-09-22T13:01:18.363 INFO:teuthology.orchestra.run.smithi169.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-22T13:01:18.363 INFO:teuthology.orchestra.run.smithi169.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-22T13:01:18.363 INFO:teuthology.orchestra.run.smithi169.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-09-22T13:01:18.363 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:01:18.364 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:01:18.477 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-mgr-rook 2023-09-22T13:01:19.335 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:15:52 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:01:20.911 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:01:20.912 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:01:20.913 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-09-22T13:01:20.913 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:01:20.913 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T13:01:20.913 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-rook noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 27 k 2023-09-22T13:01:20.913 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-09-22T13:01:20.913 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-22T13:01:20.913 INFO:teuthology.orchestra.run.smithi169.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-22T13:01:20.914 INFO:teuthology.orchestra.run.smithi169.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-22T13:01:20.914 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-09-22T13:01:20.914 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-09-22T13:01:20.914 INFO:teuthology.orchestra.run.smithi169.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-22T13:01:20.914 INFO:teuthology.orchestra.run.smithi169.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-09-22T13:01:20.914 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-09-22T13:01:20.914 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-09-22T13:01:20.915 INFO:teuthology.orchestra.run.smithi169.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-09-22T13:01:20.915 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-22T13:01:20.915 INFO:teuthology.orchestra.run.smithi169.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-22T13:01:20.915 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:01:20.915 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T13:01:20.915 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:01:20.915 INFO:teuthology.orchestra.run.smithi169.stdout:Install 13 Packages 2023-09-22T13:01:20.915 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:01:20.916 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 1.9 M 2023-09-22T13:01:20.916 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 24 M 2023-09-22T13:01:20.916 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T13:01:21.483 INFO:teuthology.orchestra.run.smithi169.stdout:(1/13): python3-jsonpointer-1.10-11.el8.noarch. 87 kB/s | 20 kB 00:00 2023-09-22T13:01:21.508 INFO:teuthology.orchestra.run.smithi169.stdout:(2/13): ceph-mgr-rook-16.2.14-5.g31c126b7.el8.n 107 kB/s | 27 kB 00:00 2023-09-22T13:01:22.291 INFO:teuthology.orchestra.run.smithi169.stdout:(3/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 26 kB/s | 27 kB 00:01 2023-09-22T13:01:25.334 INFO:teuthology.orchestra.run.smithi169.stdout:(4/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 33 kB/s | 126 kB 00:03 2023-09-22T13:01:25.925 INFO:teuthology.orchestra.run.smithi169.stdout:(5/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 43 kB/s | 155 kB 00:03 2023-09-22T13:01:26.684 INFO:teuthology.orchestra.run.smithi169.stdout:(6/13): python3-cachetools-3.1.1-4.el8.noarch.r 43 kB/s | 33 kB 00:00 2023-09-22T13:01:26.776 INFO:teuthology.orchestra.run.smithi169.stdout:(7/13): python3-certifi-2018.10.15-7.el8.noarch 178 kB/s | 16 kB 00:00 2023-09-22T13:01:26.943 INFO:teuthology.orchestra.run.smithi169.stdout:(8/13): python3-google-auth-1.1.1-10.el8.noarch 487 kB/s | 81 kB 00:00 2023-09-22T13:01:27.251 INFO:teuthology.orchestra.run.smithi169.stdout:(9/13): python3-kubernetes-11.0.0-6.el8.noarch. 3.8 MB/s | 1.2 MB 00:00 2023-09-22T13:01:27.360 INFO:teuthology.orchestra.run.smithi169.stdout:(10/13): python3-rsa-4.9-2.el8.noarch.rpm 594 kB/s | 64 kB 00:00 2023-09-22T13:01:27.477 INFO:teuthology.orchestra.run.smithi169.stdout:(11/13): python3-websocket-client-0.56.0-5.el8. 523 kB/s | 61 kB 00:00 2023-09-22T13:01:27.585 INFO:teuthology.orchestra.run.smithi169.stdout:(12/13): python3-pyasn1-modules-0.3.7-6.el8.noa 18 kB/s | 110 kB 00:06 2023-09-22T13:01:30.060 INFO:teuthology.orchestra.run.smithi169.stdout:(13/13): python3-requests-oauthlib-1.0.0-1.el8. 9.0 kB/s | 43 kB 00:04 2023-09-22T13:01:30.062 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:30.062 INFO:teuthology.orchestra.run.smithi169.stdout:Total 214 kB/s | 1.9 MB 00:09 2023-09-22T13:01:30.122 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T13:01:30.138 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T13:01:30.138 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T13:01:30.225 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T13:01:30.228 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T13:01:30.703 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T13:01:30.947 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-22T13:01:31.119 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-09-22T13:01:31.329 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-09-22T13:01:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-09-22T13:01:31.750 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-09-22T13:01:31.981 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-09-22T13:01:32.256 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-09-22T13:01:32.489 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-09-22T13:01:32.770 INFO:teuthology.orchestra.run.smithi098.stdout:(8/8): python3-scipy-1.0.0-21.module_el8.5.0+77 22 kB/s | 14 MB 10:23 2023-09-22T13:01:32.771 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:01:32.771 INFO:teuthology.orchestra.run.smithi098.stdout:Total 47 kB/s | 37 MB 13:11 2023-09-22T13:01:33.000 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T13:01:33.018 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T13:01:33.019 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T13:01:33.348 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T13:01:33.352 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T13:01:33.546 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-09-22T13:01:33.809 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-09-22T13:01:33.908 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T13:01:33.984 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-09-22T13:01:34.071 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T13:01:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-09-22T13:01:34.203 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 1/8 2023-09-22T13:01:34.301 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T13:01:34.346 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T13:01:34.595 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T13:01:34.595 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 1/13 2023-09-22T13:01:34.595 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-09-22T13:01:34.595 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-09-22T13:01:34.596 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-09-22T13:01:34.596 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-09-22T13:01:34.596 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-22T13:01:34.596 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-22T13:01:34.596 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-09-22T13:01:34.596 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-22T13:01:34.596 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-09-22T13:01:34.596 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-09-22T13:01:34.596 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-09-22T13:01:34.943 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 2/8 2023-09-22T13:01:35.129 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:01:35.278 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-09-22T13:01:35.278 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:01:35.278 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T13:01:35.278 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:01:35.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-22T13:01:35.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-22T13:01:35.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-22T13:01:35.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-22T13:01:35.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-22T13:01:35.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-22T13:01:35.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-22T13:01:35.279 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-22T13:01:35.280 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-22T13:01:35.280 INFO:teuthology.orchestra.run.smithi169.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-22T13:01:35.280 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-22T13:01:35.280 INFO:teuthology.orchestra.run.smithi169.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-22T13:01:35.280 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:01:35.280 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:01:35.396 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-mgr-cephadm 2023-09-22T13:01:35.752 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:01:35.913 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T13:01:36.252 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:16:09 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:01:36.616 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-09-22T13:01:36.965 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T13:01:37.204 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T13:01:37.842 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:01:37.843 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:01:37.843 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-09-22T13:01:37.843 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:01:37.843 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T13:01:37.843 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 116 k 2023-09-22T13:01:37.843 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-09-22T13:01:37.843 INFO:teuthology.orchestra.run.smithi169.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-09-22T13:01:37.843 INFO:teuthology.orchestra.run.smithi169.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-09-22T13:01:37.844 INFO:teuthology.orchestra.run.smithi169.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-09-22T13:01:37.844 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-09-22T13:01:37.844 INFO:teuthology.orchestra.run.smithi169.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-09-22T13:01:37.844 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:01:37.844 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T13:01:37.844 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:01:37.844 INFO:teuthology.orchestra.run.smithi169.stdout:Install 6 Packages 2023-09-22T13:01:37.844 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:01:37.845 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 5.8 M 2023-09-22T13:01:37.845 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 24 M 2023-09-22T13:01:37.845 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T13:01:38.517 INFO:teuthology.orchestra.run.smithi169.stdout:(1/6): ceph-mgr-cephadm-16.2.14-5.g31c126b7.el8 334 kB/s | 116 kB 00:00 2023-09-22T13:01:38.817 INFO:teuthology.orchestra.run.smithi169.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 71 kB/s | 21 kB 00:00 2023-09-22T13:01:38.946 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-22T13:01:40.159 INFO:teuthology.orchestra.run.smithi169.stdout:(3/6): python3-execnet-1.7.1-1.el8.noarch.rpm 265 kB/s | 355 kB 00:01 2023-09-22T13:01:40.218 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-09-22T13:01:40.251 INFO:teuthology.orchestra.run.smithi169.stdout:(4/6): python3-remoto-1.2.1-1.el8.noarch.rpm 536 kB/s | 49 kB 00:00 2023-09-22T13:01:40.395 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T13:01:40.717 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 8/8 2023-09-22T13:01:40.718 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7. 1/8 2023-09-22T13:01:40.718 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-09-22T13:01:40.718 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-09-22T13:01:40.718 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-09-22T13:01:40.718 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-09-22T13:01:40.718 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-09-22T13:01:40.718 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 7/8 2023-09-22T13:01:41.347 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 8/8 2023-09-22T13:01:41.347 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:01:41.347 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T13:01:41.348 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-diskprediction-local-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:01:41.348 INFO:teuthology.orchestra.run.smithi098.stdout: libgfortran-8.5.0-20.el8.x86_64 2023-09-22T13:01:41.348 INFO:teuthology.orchestra.run.smithi098.stdout: libquadmath-8.5.0-20.el8.x86_64 2023-09-22T13:01:41.348 INFO:teuthology.orchestra.run.smithi098.stdout: openblas-0.3.15-4.el8.x86_64 2023-09-22T13:01:41.348 INFO:teuthology.orchestra.run.smithi098.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-09-22T13:01:41.348 INFO:teuthology.orchestra.run.smithi098.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-22T13:01:41.348 INFO:teuthology.orchestra.run.smithi098.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-22T13:01:41.348 INFO:teuthology.orchestra.run.smithi098.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-09-22T13:01:41.348 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:01:41.348 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:01:41.462 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph-mgr-rook 2023-09-22T13:01:42.317 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:15:35 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:01:43.897 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:01:43.898 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:01:43.899 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repository Size 2023-09-22T13:01:43.899 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:01:43.899 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T13:01:43.899 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-rook noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 27 k 2023-09-22T13:01:43.899 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2023-09-22T13:01:43.899 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-22T13:01:43.899 INFO:teuthology.orchestra.run.smithi098.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-22T13:01:43.899 INFO:teuthology.orchestra.run.smithi098.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-22T13:01:43.900 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-09-22T13:01:43.900 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-09-22T13:01:43.900 INFO:teuthology.orchestra.run.smithi098.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-22T13:01:43.900 INFO:teuthology.orchestra.run.smithi098.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-09-22T13:01:43.900 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-09-22T13:01:43.900 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-09-22T13:01:43.900 INFO:teuthology.orchestra.run.smithi098.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-09-22T13:01:43.900 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-22T13:01:43.901 INFO:teuthology.orchestra.run.smithi098.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-22T13:01:43.901 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:01:43.901 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T13:01:43.901 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:01:43.901 INFO:teuthology.orchestra.run.smithi098.stdout:Install 13 Packages 2023-09-22T13:01:43.901 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:01:43.902 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 1.9 M 2023-09-22T13:01:43.902 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 24 M 2023-09-22T13:01:43.902 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T13:01:44.760 INFO:teuthology.orchestra.run.smithi098.stdout:(1/13): ceph-mgr-rook-16.2.14-5.g31c126b7.el8.n 161 kB/s | 27 kB 00:00 2023-09-22T13:01:45.327 INFO:teuthology.orchestra.run.smithi098.stdout:(2/13): python3-jsonpointer-1.10-11.el8.noarch. 27 kB/s | 20 kB 00:00 2023-09-22T13:01:46.152 INFO:teuthology.orchestra.run.smithi098.stdout:(3/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 17 kB/s | 27 kB 00:01 2023-09-22T13:01:49.044 INFO:teuthology.orchestra.run.smithi098.stdout:(4/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 29 kB/s | 126 kB 00:04 2023-09-22T13:01:50.244 INFO:teuthology.orchestra.run.smithi098.stdout:(5/13): python3-pyasn1-modules-0.3.7-6.el8.noar 22 kB/s | 110 kB 00:04 2023-09-22T13:01:50.328 INFO:teuthology.orchestra.run.smithi098.stdout:(6/13): python3-cachetools-3.1.1-4.el8.noarch.r 396 kB/s | 33 kB 00:00 2023-09-22T13:01:50.369 INFO:teuthology.orchestra.run.smithi098.stdout:(7/13): python3-certifi-2018.10.15-7.el8.noarch 392 kB/s | 16 kB 00:00 2023-09-22T13:01:50.445 INFO:teuthology.orchestra.run.smithi098.stdout:(8/13): python3-google-auth-1.1.1-10.el8.noarch 1.1 MB/s | 81 kB 00:00 2023-09-22T13:01:50.578 INFO:teuthology.orchestra.run.smithi098.stdout:(9/13): python3-kubernetes-11.0.0-6.el8.noarch. 8.8 MB/s | 1.2 MB 00:00 2023-09-22T13:01:50.629 INFO:teuthology.orchestra.run.smithi098.stdout:(10/13): python3-rsa-4.9-2.el8.noarch.rpm 1.3 MB/s | 64 kB 00:00 2023-09-22T13:01:50.670 INFO:teuthology.orchestra.run.smithi098.stdout:(11/13): python3-websocket-client-0.56.0-5.el8. 1.4 MB/s | 61 kB 00:00 2023-09-22T13:02:03.369 INFO:teuthology.orchestra.run.smithi169.stdout:(5/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 21 kB/s | 538 kB 00:25 2023-09-22T13:02:15.972 INFO:teuthology.orchestra.run.smithi098.stdout:(12/13): python3-requests-oauthlib-1.0.0-1.el8. 1.6 kB/s | 43 kB 00:26 2023-09-22T13:02:28.522 INFO:teuthology.orchestra.run.smithi098.stdout:(13/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 3.7 kB/s | 155 kB 00:42 2023-09-22T13:02:28.522 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:02:28.522 INFO:teuthology.orchestra.run.smithi098.stdout:Total 44 kB/s | 1.9 MB 00:44 2023-09-22T13:02:28.578 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T13:02:28.593 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T13:02:28.593 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T13:02:28.681 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T13:02:28.685 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T13:02:29.165 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T13:02:29.408 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-22T13:02:29.613 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-09-22T13:02:29.841 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-09-22T13:02:30.062 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-09-22T13:02:30.269 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-09-22T13:02:30.517 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-09-22T13:02:30.776 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-09-22T13:02:31.051 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-09-22T13:02:32.096 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-09-22T13:02:32.386 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-09-22T13:02:32.570 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-09-22T13:02:32.746 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-09-22T13:02:32.896 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T13:02:33.195 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 13/13 2023-09-22T13:02:33.195 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 1/13 2023-09-22T13:02:33.195 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-09-22T13:02:33.195 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-09-22T13:02:33.195 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-09-22T13:02:33.196 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-09-22T13:02:33.196 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-22T13:02:33.196 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-22T13:02:33.196 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-09-22T13:02:33.196 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-22T13:02:33.196 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-09-22T13:02:33.196 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-09-22T13:02:33.196 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-09-22T13:02:33.869 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-09-22T13:02:33.869 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:02:33.869 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T13:02:33.869 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-rook-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:02:33.869 INFO:teuthology.orchestra.run.smithi098.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-22T13:02:33.870 INFO:teuthology.orchestra.run.smithi098.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-22T13:02:33.870 INFO:teuthology.orchestra.run.smithi098.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-22T13:02:33.870 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-22T13:02:33.870 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-22T13:02:33.870 INFO:teuthology.orchestra.run.smithi098.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-22T13:02:33.870 INFO:teuthology.orchestra.run.smithi098.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-22T13:02:33.871 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-22T13:02:33.871 INFO:teuthology.orchestra.run.smithi098.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-22T13:02:33.871 INFO:teuthology.orchestra.run.smithi098.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-22T13:02:33.871 INFO:teuthology.orchestra.run.smithi098.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-22T13:02:33.871 INFO:teuthology.orchestra.run.smithi098.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-22T13:02:33.871 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:02:33.871 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:02:33.980 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph-mgr-cephadm 2023-09-22T13:02:34.832 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:16:27 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:02:36.374 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:02:36.375 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:02:36.375 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repository Size 2023-09-22T13:02:36.375 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:02:36.375 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T13:02:36.375 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-cephadm noarch 2:16.2.14-5.g31c126b7.el8 ceph-noarch 116 k 2023-09-22T13:02:36.375 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2023-09-22T13:02:36.375 INFO:teuthology.orchestra.run.smithi098.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-09-22T13:02:36.375 INFO:teuthology.orchestra.run.smithi098.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-09-22T13:02:36.376 INFO:teuthology.orchestra.run.smithi098.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-09-22T13:02:36.376 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-09-22T13:02:36.376 INFO:teuthology.orchestra.run.smithi098.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-09-22T13:02:36.376 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:02:36.376 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T13:02:36.376 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:02:36.376 INFO:teuthology.orchestra.run.smithi098.stdout:Install 6 Packages 2023-09-22T13:02:36.376 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:02:36.377 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 5.8 M 2023-09-22T13:02:36.377 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 24 M 2023-09-22T13:02:36.377 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T13:02:36.787 INFO:teuthology.orchestra.run.smithi098.stdout:(1/6): ceph-mgr-cephadm-16.2.14-5.g31c126b7.el8 635 kB/s | 116 kB 00:00 2023-09-22T13:02:37.037 INFO:teuthology.orchestra.run.smithi098.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 85 kB/s | 21 kB 00:00 2023-09-22T13:02:37.312 INFO:teuthology.orchestra.run.smithi098.stdout:(3/6): python3-execnet-1.7.1-1.el8.noarch.rpm 1.3 MB/s | 355 kB 00:00 2023-09-22T13:02:37.404 INFO:teuthology.orchestra.run.smithi098.stdout:(4/6): python3-remoto-1.2.1-1.el8.noarch.rpm 534 kB/s | 49 kB 00:00 2023-09-22T13:03:02.577 INFO:teuthology.orchestra.run.smithi098.stdout:(5/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 21 kB/s | 538 kB 00:25 2023-09-22T13:06:28.681 INFO:teuthology.orchestra.run.smithi169.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 17 kB/s | 4.8 MB 04:50 2023-09-22T13:06:28.683 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:06:28.683 INFO:teuthology.orchestra.run.smithi169.stdout:Total 21 kB/s | 5.8 MB 04:50 2023-09-22T13:06:28.735 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T13:06:28.746 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T13:06:28.746 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T13:06:28.846 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T13:06:28.850 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T13:06:29.174 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T13:06:29.420 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 1/6 2023-09-22T13:06:29.648 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 2/6 2023-09-22T13:06:30.545 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 3/6 2023-09-22T13:06:30.875 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 4/6 2023-09-22T13:06:31.136 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-09-22T13:06:31.311 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:06:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:06:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/6 2023-09-22T13:06:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/6 2023-09-22T13:06:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/6 2023-09-22T13:06:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 4/6 2023-09-22T13:06:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 5/6 2023-09-22T13:06:32.217 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 6/6 2023-09-22T13:06:32.217 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:32.217 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T13:06:32.217 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:06:32.217 INFO:teuthology.orchestra.run.smithi169.stdout: python3-apipkg-1.5-6.el8.noarch 2023-09-22T13:06:32.217 INFO:teuthology.orchestra.run.smithi169.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-22T13:06:32.217 INFO:teuthology.orchestra.run.smithi169.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-09-22T13:06:32.217 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-22T13:06:32.217 INFO:teuthology.orchestra.run.smithi169.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-09-22T13:06:32.217 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:32.218 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:06:32.326 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-fuse 2023-09-22T13:06:33.186 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:06 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:06:34.765 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:06:34.766 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:06:34.766 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-09-22T13:06:34.766 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:06:34.766 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T13:06:34.766 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 910 k 2023-09-22T13:06:34.766 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-09-22T13:06:34.766 INFO:teuthology.orchestra.run.smithi169.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-09-22T13:06:34.767 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:34.767 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T13:06:34.767 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:06:34.767 INFO:teuthology.orchestra.run.smithi169.stdout:Install 2 Packages 2023-09-22T13:06:34.767 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:34.767 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 993 k 2023-09-22T13:06:34.767 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 2.8 M 2023-09-22T13:06:34.768 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T13:06:35.956 INFO:teuthology.orchestra.run.smithi169.stdout:(1/2): ceph-fuse-16.2.14-5.g31c126b7.el8.x86_64 907 kB/s | 910 kB 00:01 2023-09-22T13:06:37.907 INFO:teuthology.orchestra.run.smithi169.stdout:(2/2): fuse-2.9.7-17.el8.x86_64.rpm 28 kB/s | 83 kB 00:02 2023-09-22T13:06:37.907 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:06:37.907 INFO:teuthology.orchestra.run.smithi169.stdout:Total 316 kB/s | 993 kB 00:03 2023-09-22T13:06:37.919 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T13:06:37.941 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T13:06:37.941 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T13:06:38.032 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T13:06:38.034 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T13:06:38.203 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T13:06:38.459 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-09-22T13:06:38.660 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:06:39.046 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:06:39.046 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/2 2023-09-22T13:06:39.703 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-09-22T13:06:39.703 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:39.703 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T13:06:39.703 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-09-22T13:06:39.703 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:39.704 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:06:39.902 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install librados-devel 2023-09-22T13:06:40.755 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:13 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:06:42.326 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:06:42.327 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:06:42.327 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repo Size 2023-09-22T13:06:42.327 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:06:42.327 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T13:06:42.327 INFO:teuthology.orchestra.run.smithi169.stdout: librados-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 131 k 2023-09-22T13:06:42.327 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:42.327 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T13:06:42.327 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:06:42.327 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-09-22T13:06:42.327 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:42.328 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 131 k 2023-09-22T13:06:42.328 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 432 k 2023-09-22T13:06:42.328 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T13:06:42.592 INFO:teuthology.orchestra.run.smithi169.stdout:librados-devel-16.2.14-5.g31c126b7.el8.x86_64.r 495 kB/s | 131 kB 00:00 2023-09-22T13:06:42.593 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:06:42.593 INFO:teuthology.orchestra.run.smithi169.stdout:Total 493 kB/s | 131 kB 00:00 2023-09-22T13:06:42.594 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T13:06:42.599 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T13:06:42.599 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T13:06:42.647 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T13:06:42.651 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T13:06:42.758 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T13:06:42.975 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:06:43.267 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:06:43.879 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:06:43.879 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:43.880 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T13:06:43.880 INFO:teuthology.orchestra.run.smithi169.stdout: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:06:43.880 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:43.880 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:06:44.096 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install libcephfs2 2023-09-22T13:06:44.960 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:17 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:06:46.328 INFO:teuthology.orchestra.run.smithi169.stdout:Package libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:06:46.511 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:06:46.512 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-09-22T13:06:46.512 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:06:46.597 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install libcephfs-devel 2023-09-22T13:06:47.457 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:20 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:06:49.039 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:06:49.040 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:06:49.040 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repo Size 2023-09-22T13:06:49.040 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:06:49.040 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T13:06:49.041 INFO:teuthology.orchestra.run.smithi169.stdout: libcephfs-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 25 k 2023-09-22T13:06:49.041 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:49.041 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T13:06:49.041 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:06:49.041 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-09-22T13:06:49.041 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:49.041 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 25 k 2023-09-22T13:06:49.041 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 107 k 2023-09-22T13:06:49.042 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T13:06:49.479 INFO:teuthology.orchestra.run.smithi169.stdout:libcephfs-devel-16.2.14-5.g31c126b7.el8.x86_64. 58 kB/s | 25 kB 00:00 2023-09-22T13:06:49.480 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:06:49.480 INFO:teuthology.orchestra.run.smithi169.stdout:Total 58 kB/s | 25 kB 00:00 2023-09-22T13:06:49.480 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T13:06:49.483 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T13:06:49.484 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T13:06:49.486 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T13:06:49.489 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T13:06:49.542 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T13:06:49.732 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:06:50.007 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:06:50.160 INFO:teuthology.orchestra.run.smithi098.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 19 kB/s | 4.8 MB 04:13 2023-09-22T13:06:50.162 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:06:50.163 INFO:teuthology.orchestra.run.smithi098.stdout:Total 24 kB/s | 5.8 MB 04:13 2023-09-22T13:06:50.214 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T13:06:50.224 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T13:06:50.224 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T13:06:50.320 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T13:06:50.324 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T13:06:50.586 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:06:50.587 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:50.587 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T13:06:50.587 INFO:teuthology.orchestra.run.smithi169.stdout: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:06:50.587 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:06:50.587 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:06:50.643 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T13:06:50.693 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install librados2 2023-09-22T13:06:50.919 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 1/6 2023-09-22T13:06:51.206 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 2/6 2023-09-22T13:06:51.551 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:24 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:06:52.146 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 3/6 2023-09-22T13:06:52.498 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 4/6 2023-09-22T13:06:52.753 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-09-22T13:06:52.945 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:06:52.957 INFO:teuthology.orchestra.run.smithi169.stdout:Package librados2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:06:53.138 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:06:53.139 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-09-22T13:06:53.139 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:06:53.233 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 6/6 2023-09-22T13:06:53.233 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 1/6 2023-09-22T13:06:53.233 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/6 2023-09-22T13:06:53.233 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/6 2023-09-22T13:06:53.233 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 4/6 2023-09-22T13:06:53.233 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 5/6 2023-09-22T13:06:53.302 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install librbd1 2023-09-22T13:06:53.847 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 6/6 2023-09-22T13:06:53.847 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:06:53.847 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T13:06:53.847 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-mgr-cephadm-2:16.2.14-5.g31c126b7.el8.noarch 2023-09-22T13:06:53.847 INFO:teuthology.orchestra.run.smithi098.stdout: python3-apipkg-1.5-6.el8.noarch 2023-09-22T13:06:53.847 INFO:teuthology.orchestra.run.smithi098.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-22T13:06:53.848 INFO:teuthology.orchestra.run.smithi098.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-09-22T13:06:53.848 INFO:teuthology.orchestra.run.smithi098.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-22T13:06:53.848 INFO:teuthology.orchestra.run.smithi098.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-09-22T13:06:53.848 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:06:53.848 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:06:53.945 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install ceph-fuse 2023-09-22T13:06:54.162 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:27 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:06:54.800 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:20:47 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:06:55.556 INFO:teuthology.orchestra.run.smithi169.stdout:Package librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:06:55.738 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:06:55.740 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-09-22T13:06:55.740 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:06:55.912 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install python3-rados 2023-09-22T13:06:56.375 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:06:56.375 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:06:56.376 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repository Size 2023-09-22T13:06:56.376 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:06:56.376 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T13:06:56.376 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 910 k 2023-09-22T13:06:56.376 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2023-09-22T13:06:56.376 INFO:teuthology.orchestra.run.smithi098.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-09-22T13:06:56.376 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:06:56.376 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T13:06:56.377 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:06:56.377 INFO:teuthology.orchestra.run.smithi098.stdout:Install 2 Packages 2023-09-22T13:06:56.377 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:06:56.377 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 993 k 2023-09-22T13:06:56.378 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 2.8 M 2023-09-22T13:06:56.378 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T13:06:56.767 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:29 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:06:56.845 INFO:teuthology.orchestra.run.smithi098.stdout:(1/2): ceph-fuse-16.2.14-5.g31c126b7.el8.x86_64 2.3 MB/s | 910 kB 00:00 2023-09-22T13:06:56.995 INFO:teuthology.orchestra.run.smithi098.stdout:(2/2): fuse-2.9.7-17.el8.x86_64.rpm 154 kB/s | 83 kB 00:00 2023-09-22T13:06:56.996 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:06:56.996 INFO:teuthology.orchestra.run.smithi098.stdout:Total 1.6 MB/s | 993 kB 00:00 2023-09-22T13:06:57.006 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T13:06:57.027 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T13:06:57.027 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T13:06:57.113 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T13:06:57.116 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T13:06:57.283 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T13:06:57.552 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-09-22T13:06:57.769 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:06:58.147 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2/2 2023-09-22T13:06:58.149 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/2 2023-09-22T13:06:58.151 INFO:teuthology.orchestra.run.smithi169.stdout:Package python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:06:58.330 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:06:58.332 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-09-22T13:06:58.332 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:06:58.518 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install python3-rgw 2023-09-22T13:06:58.796 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-09-22T13:06:58.796 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:06:58.796 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T13:06:58.797 INFO:teuthology.orchestra.run.smithi098.stdout: ceph-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-09-22T13:06:58.797 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:06:58.797 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:06:59.001 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install librados-devel 2023-09-22T13:06:59.371 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:32 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:06:59.858 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:20:52 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:00.774 INFO:teuthology.orchestra.run.smithi169.stdout:Package python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:07:00.955 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:07:00.957 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-09-22T13:07:00.957 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:07:01.052 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install python3-cephfs 2023-09-22T13:07:01.439 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:01.439 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:01.439 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repo Size 2023-09-22T13:07:01.439 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:01.439 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T13:07:01.440 INFO:teuthology.orchestra.run.smithi098.stdout: librados-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 131 k 2023-09-22T13:07:01.440 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:01.440 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T13:07:01.440 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:01.440 INFO:teuthology.orchestra.run.smithi098.stdout:Install 1 Package 2023-09-22T13:07:01.440 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:01.440 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 131 k 2023-09-22T13:07:01.441 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 432 k 2023-09-22T13:07:01.441 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T13:07:01.623 INFO:teuthology.orchestra.run.smithi098.stdout:librados-devel-16.2.14-5.g31c126b7.el8.x86_64.r 717 kB/s | 131 kB 00:00 2023-09-22T13:07:01.624 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:07:01.624 INFO:teuthology.orchestra.run.smithi098.stdout:Total 712 kB/s | 131 kB 00:00 2023-09-22T13:07:01.624 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T13:07:01.629 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T13:07:01.630 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T13:07:01.678 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T13:07:01.681 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T13:07:01.788 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T13:07:01.913 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:34 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:07:02.001 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:02.298 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:02.887 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:02.887 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:02.887 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T13:07:02.887 INFO:teuthology.orchestra.run.smithi098.stdout: librados-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:02.887 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:02.887 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:03.082 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install libcephfs2 2023-09-22T13:07:03.307 INFO:teuthology.orchestra.run.smithi169.stdout:Package python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:07:03.487 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:07:03.489 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-09-22T13:07:03.489 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:07:03.661 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install python3-rbd 2023-09-22T13:07:03.937 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:20:56 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:04.522 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:37 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:07:05.338 INFO:teuthology.orchestra.run.smithi098.stdout:Package libcephfs2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:07:05.518 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:05.519 INFO:teuthology.orchestra.run.smithi098.stdout:Nothing to do. 2023-09-22T13:07:05.519 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:05.607 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install libcephfs-devel 2023-09-22T13:07:05.938 INFO:teuthology.orchestra.run.smithi169.stdout:Package python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:07:06.118 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:07:06.119 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-09-22T13:07:06.119 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:07:06.307 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install rbd-fuse 2023-09-22T13:07:06.459 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:20:59 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:07.165 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:40 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:07:08.023 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:08.024 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:08.024 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repo Size 2023-09-22T13:07:08.025 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:08.025 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T13:07:08.025 INFO:teuthology.orchestra.run.smithi098.stdout: libcephfs-devel x86_64 2:16.2.14-5.g31c126b7.el8 ceph 25 k 2023-09-22T13:07:08.025 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:08.025 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T13:07:08.025 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:08.025 INFO:teuthology.orchestra.run.smithi098.stdout:Install 1 Package 2023-09-22T13:07:08.026 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:08.026 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 25 k 2023-09-22T13:07:08.026 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 107 k 2023-09-22T13:07:08.026 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T13:07:08.318 INFO:teuthology.orchestra.run.smithi098.stdout:libcephfs-devel-16.2.14-5.g31c126b7.el8.x86_64. 87 kB/s | 25 kB 00:00 2023-09-22T13:07:08.318 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:07:08.318 INFO:teuthology.orchestra.run.smithi098.stdout:Total 87 kB/s | 25 kB 00:00 2023-09-22T13:07:08.319 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T13:07:08.322 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T13:07:08.322 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T13:07:08.324 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T13:07:08.327 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T13:07:08.381 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T13:07:08.584 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:08.773 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:07:08.774 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:07:08.774 INFO:teuthology.orchestra.run.smithi169.stdout: Package Architecture Version Repository Size 2023-09-22T13:07:08.774 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:07:08.774 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T13:07:08.774 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 88 k 2023-09-22T13:07:08.774 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:08.775 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T13:07:08.775 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:07:08.775 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-09-22T13:07:08.775 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:08.775 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 88 k 2023-09-22T13:07:08.775 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 240 k 2023-09-22T13:07:08.776 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T13:07:08.852 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:09.032 INFO:teuthology.orchestra.run.smithi169.stdout:rbd-fuse-16.2.14-5.g31c126b7.el8.x86_64.rpm 344 kB/s | 88 kB 00:00 2023-09-22T13:07:09.033 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:07:09.033 INFO:teuthology.orchestra.run.smithi169.stdout:Total 342 kB/s | 88 kB 00:00 2023-09-22T13:07:09.033 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T13:07:09.043 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T13:07:09.043 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T13:07:09.093 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T13:07:09.096 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T13:07:09.200 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T13:07:09.404 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:09.421 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:09.421 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:09.421 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T13:07:09.421 INFO:teuthology.orchestra.run.smithi098.stdout: libcephfs-devel-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:09.422 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:09.422 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:09.522 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install librados2 2023-09-22T13:07:09.697 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:10.314 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:10.314 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:10.314 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T13:07:10.314 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:10.315 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:10.315 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:07:10.371 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:21:03 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:10.508 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install rbd-mirror 2023-09-22T13:07:11.366 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:44 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:07:11.777 INFO:teuthology.orchestra.run.smithi098.stdout:Package librados2-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:07:11.957 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:11.958 INFO:teuthology.orchestra.run.smithi098.stdout:Nothing to do. 2023-09-22T13:07:11.959 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:12.114 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install librbd1 2023-09-22T13:07:12.950 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:07:12.951 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:07:12.951 INFO:teuthology.orchestra.run.smithi169.stdout: Package Architecture Version Repository Size 2023-09-22T13:07:12.951 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:07:12.951 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T13:07:12.951 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-mirror x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T13:07:12.951 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:12.952 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T13:07:12.952 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:07:12.952 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-09-22T13:07:12.952 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:12.952 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 3.7 M 2023-09-22T13:07:12.952 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 15 M 2023-09-22T13:07:12.952 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T13:07:12.969 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:21:05 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:13.660 INFO:teuthology.orchestra.run.smithi169.stdout:rbd-mirror-16.2.14-5.g31c126b7.el8.x86_64.rpm 5.2 MB/s | 3.7 MB 00:00 2023-09-22T13:07:13.661 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:07:13.662 INFO:teuthology.orchestra.run.smithi169.stdout:Total 5.2 MB/s | 3.7 MB 00:00 2023-09-22T13:07:13.662 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T13:07:13.674 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T13:07:13.674 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T13:07:13.741 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T13:07:13.744 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T13:07:14.189 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T13:07:14.357 INFO:teuthology.orchestra.run.smithi098.stdout:Package librbd1-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:07:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:14.538 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:14.540 INFO:teuthology.orchestra.run.smithi098.stdout:Nothing to do. 2023-09-22T13:07:14.540 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:14.721 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install python3-rados 2023-09-22T13:07:14.791 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:15.411 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:15.411 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:15.411 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T13:07:15.411 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:15.411 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:15.411 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:07:15.579 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:21:08 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:15.633 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install rbd-nbd 2023-09-22T13:07:16.500 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:21:49 ago on Fri 22 Sep 2023 12:45:27 PM UTC. 2023-09-22T13:07:16.972 INFO:teuthology.orchestra.run.smithi098.stdout:Package python3-rados-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:07:17.153 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:17.154 INFO:teuthology.orchestra.run.smithi098.stdout:Nothing to do. 2023-09-22T13:07:17.154 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:17.337 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install python3-rgw 2023-09-22T13:07:18.077 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-09-22T13:07:18.078 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:07:18.078 INFO:teuthology.orchestra.run.smithi169.stdout: Package Architecture Version Repository Size 2023-09-22T13:07:18.078 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:07:18.078 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-09-22T13:07:18.078 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-nbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 173 k 2023-09-22T13:07:18.078 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:18.078 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-09-22T13:07:18.079 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-09-22T13:07:18.079 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-09-22T13:07:18.079 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:18.080 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 173 k 2023-09-22T13:07:18.080 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 515 k 2023-09-22T13:07:18.080 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-09-22T13:07:18.186 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:21:11 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:18.413 INFO:teuthology.orchestra.run.smithi169.stdout:rbd-nbd-16.2.14-5.g31c126b7.el8.x86_64.rpm 520 kB/s | 173 kB 00:00 2023-09-22T13:07:18.413 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:07:18.414 INFO:teuthology.orchestra.run.smithi169.stdout:Total 518 kB/s | 173 kB 00:00 2023-09-22T13:07:18.414 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-09-22T13:07:18.424 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-09-22T13:07:18.424 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-09-22T13:07:18.475 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-09-22T13:07:18.478 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-09-22T13:07:18.589 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-09-22T13:07:18.761 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:19.048 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:19.566 INFO:teuthology.orchestra.run.smithi098.stdout:Package python3-rgw-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:07:19.664 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:19.664 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:19.664 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-09-22T13:07:19.665 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:19.665 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:19.665 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-09-22T13:07:19.745 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:19.747 INFO:teuthology.orchestra.run.smithi098.stdout:Nothing to do. 2023-09-22T13:07:19.747 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:19.832 DEBUG:teuthology.parallel:result is None 2023-09-22T13:07:19.835 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install python3-cephfs 2023-09-22T13:07:20.683 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:21:13 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:22.063 INFO:teuthology.orchestra.run.smithi098.stdout:Package python3-cephfs-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:07:22.244 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:22.245 INFO:teuthology.orchestra.run.smithi098.stdout:Nothing to do. 2023-09-22T13:07:22.245 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:22.419 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install python3-rbd 2023-09-22T13:07:23.270 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:21:16 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:24.657 INFO:teuthology.orchestra.run.smithi098.stdout:Package python3-rbd-2:16.2.14-5.g31c126b7.el8.x86_64 is already installed. 2023-09-22T13:07:24.837 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:24.839 INFO:teuthology.orchestra.run.smithi098.stdout:Nothing to do. 2023-09-22T13:07:24.839 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:24.929 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install rbd-fuse 2023-09-22T13:07:25.784 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:21:18 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:27.352 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:27.354 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:27.354 INFO:teuthology.orchestra.run.smithi098.stdout: Package Architecture Version Repository Size 2023-09-22T13:07:27.354 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:27.354 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T13:07:27.354 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-fuse x86_64 2:16.2.14-5.g31c126b7.el8 ceph 88 k 2023-09-22T13:07:27.354 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:27.355 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T13:07:27.355 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:27.355 INFO:teuthology.orchestra.run.smithi098.stdout:Install 1 Package 2023-09-22T13:07:27.355 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:27.355 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 88 k 2023-09-22T13:07:27.355 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 240 k 2023-09-22T13:07:27.355 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T13:07:27.513 INFO:teuthology.orchestra.run.smithi098.stdout:rbd-fuse-16.2.14-5.g31c126b7.el8.x86_64.rpm 556 kB/s | 88 kB 00:00 2023-09-22T13:07:27.514 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:07:27.514 INFO:teuthology.orchestra.run.smithi098.stdout:Total 551 kB/s | 88 kB 00:00 2023-09-22T13:07:27.515 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T13:07:27.524 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T13:07:27.524 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T13:07:27.573 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T13:07:27.576 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T13:07:27.680 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T13:07:27.889 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:28.176 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:28.770 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:28.770 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:28.770 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T13:07:28.770 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-fuse-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:28.771 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:28.771 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:28.951 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install rbd-mirror 2023-09-22T13:07:29.801 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:21:22 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:31.387 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:31.388 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:31.388 INFO:teuthology.orchestra.run.smithi098.stdout: Package Architecture Version Repository Size 2023-09-22T13:07:31.388 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:31.388 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T13:07:31.388 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-mirror x86_64 2:16.2.14-5.g31c126b7.el8 ceph 3.7 M 2023-09-22T13:07:31.388 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:31.388 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T13:07:31.388 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:31.389 INFO:teuthology.orchestra.run.smithi098.stdout:Install 1 Package 2023-09-22T13:07:31.389 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:31.389 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 3.7 M 2023-09-22T13:07:31.389 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 15 M 2023-09-22T13:07:31.389 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T13:07:32.025 INFO:teuthology.orchestra.run.smithi098.stdout:rbd-mirror-16.2.14-5.g31c126b7.el8.x86_64.rpm 5.8 MB/s | 3.7 MB 00:00 2023-09-22T13:07:32.026 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:07:32.026 INFO:teuthology.orchestra.run.smithi098.stdout:Total 5.8 MB/s | 3.7 MB 00:00 2023-09-22T13:07:32.026 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T13:07:32.038 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T13:07:32.038 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T13:07:32.103 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T13:07:32.107 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T13:07:32.553 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T13:07:32.765 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:33.184 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:33.766 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:33.766 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:33.766 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T13:07:33.766 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-mirror-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:33.767 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:33.767 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:33.936 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum -y install rbd-nbd 2023-09-22T13:07:34.792 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:21:27 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:07:36.365 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:07:36.366 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:36.367 INFO:teuthology.orchestra.run.smithi098.stdout: Package Architecture Version Repository Size 2023-09-22T13:07:36.367 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:36.367 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T13:07:36.367 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-nbd x86_64 2:16.2.14-5.g31c126b7.el8 ceph 173 k 2023-09-22T13:07:36.367 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:36.367 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T13:07:36.367 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:07:36.367 INFO:teuthology.orchestra.run.smithi098.stdout:Install 1 Package 2023-09-22T13:07:36.368 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:36.368 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 173 k 2023-09-22T13:07:36.368 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 515 k 2023-09-22T13:07:36.368 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T13:07:36.554 INFO:teuthology.orchestra.run.smithi098.stdout:rbd-nbd-16.2.14-5.g31c126b7.el8.x86_64.rpm 932 kB/s | 173 kB 00:00 2023-09-22T13:07:36.555 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:07:36.555 INFO:teuthology.orchestra.run.smithi098.stdout:Total 925 kB/s | 173 kB 00:00 2023-09-22T13:07:36.555 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T13:07:36.565 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T13:07:36.565 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T13:07:36.614 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T13:07:36.617 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T13:07:36.730 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T13:07:36.913 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:37.193 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:37.799 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 1/1 2023-09-22T13:07:37.799 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:37.799 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T13:07:37.799 INFO:teuthology.orchestra.run.smithi098.stdout: rbd-nbd-2:16.2.14-5.g31c126b7.el8.x86_64 2023-09-22T13:07:37.799 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:37.800 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:07:37.991 DEBUG:teuthology.parallel:result is None 2023-09-22T13:07:37.992 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:07:38.234 DEBUG:teuthology.orchestra.run.smithi098:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-22T13:07:38.257 INFO:teuthology.orchestra.run.smithi098.stdout:16.2.14-5.g31c126b7.el8 2023-09-22T13:07:38.258 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-5.g31c126b7.el8 2023-09-22T13:07:38.258 INFO:teuthology.task.install:The correct ceph version 16.2.14-5.g31c126b7 is installed. 2023-09-22T13:07:38.258 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:07:38.493 DEBUG:teuthology.orchestra.run.smithi169:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-22T13:07:38.523 INFO:teuthology.orchestra.run.smithi169.stdout:16.2.14-5.g31c126b7.el8 2023-09-22T13:07:38.523 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-5.g31c126b7.el8 2023-09-22T13:07:38.523 INFO:teuthology.task.install:The correct ceph version 16.2.14-5.g31c126b7 is installed. 2023-09-22T13:07:38.524 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-22T13:07:38.524 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:38.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-22T13:07:38.554 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:38.554 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-22T13:07:38.597 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-22T13:07:38.597 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:38.597 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/usr/bin/daemon-helper 2023-09-22T13:07:38.624 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-22T13:07:38.692 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:38.692 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/usr/bin/daemon-helper 2023-09-22T13:07:38.722 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-22T13:07:38.792 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-22T13:07:38.792 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:38.792 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-22T13:07:38.820 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-22T13:07:38.886 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:38.887 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-22T13:07:38.916 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-22T13:07:38.985 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-22T13:07:38.985 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:38.986 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/usr/bin/stdin-killer 2023-09-22T13:07:39.012 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-22T13:07:39.080 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:39.080 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/usr/bin/stdin-killer 2023-09-22T13:07:39.108 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-22T13:07:39.177 INFO:teuthology.run_tasks:Running task ceph... 2023-09-22T13:07:39.236 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2023-09-22T13:07:39.236 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 777 /var/log/ceph 2023-09-22T13:07:39.239 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 777 /var/log/ceph 2023-09-22T13:07:39.266 INFO:tasks.ceph:Disabling ceph logrotate... 2023-09-22T13:07:39.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -f -- /etc/logrotate.d/ceph 2023-09-22T13:07:39.306 DEBUG:teuthology.orchestra.run.smithi169:> sudo rm -f -- /etc/logrotate.d/ceph 2023-09-22T13:07:39.334 INFO:tasks.ceph:Creating extra log directories... 2023-09-22T13:07:39.334 DEBUG:teuthology.orchestra.run.smithi098:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2023-09-22T13:07:39.373 DEBUG:teuthology.orchestra.run.smithi169:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2023-09-22T13:07:39.412 INFO:tasks.ceph:Creating ceph cluster ceph... 2023-09-22T13:07:39.413 INFO:tasks.ceph:config {'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=', 'rgw lc debug interval': 10, 'rgw max chunk size': 4194304, 'rgw obj stripe size': 6291456, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'global': {'mon client directed command retry': 5, 'ms inject socket failures': 5000, 'osd_max_pg_log_entries': 10, 'osd_min_pg_log_entries': 10, 'rgw override bucket index max shards': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'lockdep': True}, 'osd': {'debug ms': 1, 'debug osd': 20, 'lockdep': True, 'osd objectstore': 'filestore', 'osd shutdown pgref assert': True, 'osd sloppy crc': True}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(POOL_APP_NOT_ENABLED\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'ceph', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2023-09-22T13:07:39.413 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi/7400908', 'branch': 'wip-cfsnyder-bucket-check-pacific', 'description': 'rgw/verify/{centos_latest clusters/fixed-2 frontend/beast ignore-pg-availability msgr-failures/few objectstore/filestore-xfs overrides proto/http rgw_pool_type/ec-profile sharding$/{single} striping$/{stripe-greater-than-chunk} tasks/{0-install bucket-check cls ragweed reshard s3tests-java s3tests versioning} validater/lockdep}', 'email': None, 'first_in_suite': False, 'job_id': '7400908', 'kernel': {'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'name': 'cfsnyder-2023-09-22_11:28:34-rgw-wip-cfsnyder-bucket-check-pacific-distro-default-smithi', 'no_nested_subset': False, 'nuke-on-error': True, 'openstack': [{'volumes': {'count': 4, 'size': 10}}], 'os_type': 'centos', 'os_version': '8.stream', 'overrides': {'admin_socket': {'branch': 'wip-cfsnyder-bucket-check-pacific'}, 'ceph': {'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=', 'rgw lc debug interval': 10, 'rgw max chunk size': 4194304, 'rgw obj stripe size': 6291456, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'global': {'mon client directed command retry': 5, 'ms inject socket failures': 5000, 'osd_max_pg_log_entries': 10, 'osd_min_pg_log_entries': 10, 'rgw override bucket index max shards': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'lockdep': True}, 'osd': {'debug ms': 1, 'debug osd': 20, 'lockdep': True, 'osd objectstore': 'filestore', 'osd shutdown pgref assert': True, 'osd sloppy crc': True}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)', '\\(POOL_APP_NOT_ENABLED\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {'osd default pool size': 2}, 'osd': {'osd objectstore': 'filestore', 'osd sloppy crc': True}}, 'filestore': True, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}}, 'rgw': {'compression type': 'random', 'ec-data-pool': True, 'erasure_code_profile': {'crush-failure-domain': 'osd', 'k': 3, 'm': 1, 'name': 'testprofile'}, 'frontend': 'beast', 'storage classes': 'LUKEWARM, FROZEN'}, 's3tests': {'slow_backend': True}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'wip-cfsnyder-bucket-check-pacific', 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'}}, 'owner': 'scheduled_cfsnyder@teuthology', 'priority': 101, 'repo': 'https://git.ceph.com/ceph-ci.git', 'roles': [['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a'], ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'node-exporter.b']], 'sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'sleep_before_teardown': 0, 'suite': 'rgw', 'suite_branch': 'wip-cfsnyder-bucket-check-pacific', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph-c_31c126b7c1c0c06b66d9d19c5bfc51c46b503392/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph-ci.git', 'suite_sha1': '31c126b7c1c0c06b66d9d19c5bfc51c46b503392', 'targets': {'smithi098.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEnG2dpCSqF0hcwU4l2Bou9zeEz6+Sp3++/ejmApHnc/7fEtxPmKzXNnoR3W7l2Wlnu0AptzOM6FtHOEMLGLUKA=', 'smithi169.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOG6FCbq8Q3wdwSRGUW36jmNWVrtbiundCdeWgGKR9ILzOBTbsBqOr1p7aHTZ6nXFncYlVlXSlcuwgzrlu9yxuI='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': None}, {'ceph': None}, {'openssl_keys': None}, {'rgw': {'client.0': None}}, {'workunit': {'clients': {'client.0': ['rgw/run-bucket-check.sh']}}}, {'workunit': {'clients': {'client.0': ['cls/test_cls_lock.sh', 'cls/test_cls_log.sh', 'cls/test_cls_refcount.sh', 'cls/test_cls_rgw.sh', 'cls/test_cls_cmpomap.sh', 'cls/test_cls_2pc_queue.sh', 'rgw/test_rgw_gc_log.sh', 'rgw/test_rgw_obj.sh', 'rgw/test_rgw_throttle.sh', 'rgw/test_librgw_file.sh']}}}, {'ragweed': {'client.0': {'default-branch': 'ceph-pacific', 'rgw_server': 'client.0', 'stages': 'prepare,check'}}}, {'workunit': {'clients': {'client.0': ['rgw/run-reshard.sh']}}}, {'s3tests-java': {'client.0': {'force-branch': 'ceph-pacific', 'force-repo': 'https://github.com/ceph/java_s3tests.git'}}}, {'s3tests': {'client.0': {'force-branch': 'ceph-pacific', 'rgw_server': 'client.0'}}}, {'workunit': {'clients': {'client.0': ['rgw/run-versioning.sh']}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '54e62bcbac4e53d9685e08328b790d3b20d71cae', 'timestamp': '2023-09-22_11:28:34', 'tube': 'smithi', 'user': 'cfsnyder', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.2830'} 2023-09-22T13:07:39.414 DEBUG:teuthology.orchestra.run.smithi098:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2023-09-22T13:07:39.441 DEBUG:teuthology.orchestra.run.smithi169:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2023-09-22T13:07:39.461 DEBUG:teuthology.orchestra.run.smithi098:> sudo install -d -m0777 -- /var/run/ceph 2023-09-22T13:07:39.500 DEBUG:teuthology.orchestra.run.smithi169:> sudo install -d -m0777 -- /var/run/ceph 2023-09-22T13:07:39.529 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:39.529 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/scratch_devs of=/dev/stdout 2023-09-22T13:07:39.584 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-22T13:07:39.584 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_1 2023-09-22T13:07:39.641 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-22T13:07:39.641 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:07:39.642 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 6h/6d Inode: 423046 Links: 1 2023-09-22T13:07:39.642 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:07:39.642 INFO:teuthology.orchestra.run.smithi098.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:07:39.642 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2023-09-22 13:07:32.937699172 +0000 2023-09-22T13:07:39.642 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2023-09-22 12:17:07.852177044 +0000 2023-09-22T13:07:39.642 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2023-09-22 12:17:07.852177044 +0000 2023-09-22T13:07:39.642 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: - 2023-09-22T13:07:39.643 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-22T13:07:39.707 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2023-09-22T13:07:39.708 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2023-09-22T13:07:39.708 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000231345 s, 2.2 MB/s 2023-09-22T13:07:39.709 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-22T13:07:39.767 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_2 2023-09-22T13:07:39.825 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-22T13:07:39.826 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:07:39.826 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 6h/6d Inode: 421336 Links: 1 2023-09-22T13:07:39.826 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:07:39.826 INFO:teuthology.orchestra.run.smithi098.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:07:39.826 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2023-09-22 13:07:32.937699172 +0000 2023-09-22T13:07:39.826 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2023-09-22 12:17:08.354171003 +0000 2023-09-22T13:07:39.826 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2023-09-22 12:17:08.354171003 +0000 2023-09-22T13:07:39.826 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: - 2023-09-22T13:07:39.827 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-22T13:07:39.892 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2023-09-22T13:07:39.892 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2023-09-22T13:07:39.892 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000219882 s, 2.3 MB/s 2023-09-22T13:07:39.893 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-22T13:07:39.951 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_3 2023-09-22T13:07:40.010 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-22T13:07:40.010 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:07:40.010 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 6h/6d Inode: 425135 Links: 1 2023-09-22T13:07:40.010 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:07:40.011 INFO:teuthology.orchestra.run.smithi098.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:07:40.011 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2023-09-22 13:07:32.937699172 +0000 2023-09-22T13:07:40.011 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2023-09-22 12:17:08.844165107 +0000 2023-09-22T13:07:40.011 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2023-09-22 12:17:08.844165107 +0000 2023-09-22T13:07:40.011 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: - 2023-09-22T13:07:40.011 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-22T13:07:40.077 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2023-09-22T13:07:40.077 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2023-09-22T13:07:40.077 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000229298 s, 2.2 MB/s 2023-09-22T13:07:40.078 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-22T13:07:40.137 DEBUG:teuthology.orchestra.run.smithi098:> stat /dev/vg_nvme/lv_4 2023-09-22T13:07:40.196 INFO:teuthology.orchestra.run.smithi098.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-22T13:07:40.196 INFO:teuthology.orchestra.run.smithi098.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:07:40.196 INFO:teuthology.orchestra.run.smithi098.stdout:Device: 6h/6d Inode: 426330 Links: 1 2023-09-22T13:07:40.196 INFO:teuthology.orchestra.run.smithi098.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:07:40.196 INFO:teuthology.orchestra.run.smithi098.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:07:40.197 INFO:teuthology.orchestra.run.smithi098.stdout:Access: 2023-09-22 13:07:32.937699172 +0000 2023-09-22T13:07:40.197 INFO:teuthology.orchestra.run.smithi098.stdout:Modify: 2023-09-22 12:17:09.333159144 +0000 2023-09-22T13:07:40.197 INFO:teuthology.orchestra.run.smithi098.stdout:Change: 2023-09-22 12:17:09.333159144 +0000 2023-09-22T13:07:40.197 INFO:teuthology.orchestra.run.smithi098.stdout: Birth: - 2023-09-22T13:07:40.197 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-22T13:07:40.261 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records in 2023-09-22T13:07:40.261 INFO:teuthology.orchestra.run.smithi098.stderr:1+0 records out 2023-09-22T13:07:40.261 INFO:teuthology.orchestra.run.smithi098.stderr:512 bytes copied, 0.000233217 s, 2.2 MB/s 2023-09-22T13:07:40.263 DEBUG:teuthology.orchestra.run.smithi098:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-22T13:07:40.321 INFO:tasks.ceph:osd dev map: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:07:40.322 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:40.322 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/scratch_devs of=/dev/stdout 2023-09-22T13:07:40.343 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-22T13:07:40.343 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_1 2023-09-22T13:07:40.404 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-22T13:07:40.404 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:07:40.404 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 6h/6d Inode: 419224 Links: 1 2023-09-22T13:07:40.404 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:07:40.405 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:07:40.405 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2023-09-22 13:07:14.538422390 +0000 2023-09-22T13:07:40.405 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2023-09-22 12:16:15.223139927 +0000 2023-09-22T13:07:40.405 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2023-09-22 12:16:15.223139927 +0000 2023-09-22T13:07:40.405 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2023-09-22T13:07:40.405 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-22T13:07:40.473 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2023-09-22T13:07:40.474 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2023-09-22T13:07:40.474 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000249997 s, 2.0 MB/s 2023-09-22T13:07:40.475 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-22T13:07:40.537 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_2 2023-09-22T13:07:40.596 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-22T13:07:40.596 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:07:40.596 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 6h/6d Inode: 424961 Links: 1 2023-09-22T13:07:40.597 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:07:40.597 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:07:40.597 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2023-09-22 13:07:14.538422390 +0000 2023-09-22T13:07:40.597 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2023-09-22 12:16:15.735129428 +0000 2023-09-22T13:07:40.597 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2023-09-22 12:16:15.735129428 +0000 2023-09-22T13:07:40.597 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2023-09-22T13:07:40.598 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-22T13:07:40.665 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2023-09-22T13:07:40.665 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2023-09-22T13:07:40.665 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000244301 s, 2.1 MB/s 2023-09-22T13:07:40.666 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-22T13:07:40.727 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_3 2023-09-22T13:07:40.786 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-22T13:07:40.786 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:07:40.786 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 6h/6d Inode: 427029 Links: 1 2023-09-22T13:07:40.786 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:07:40.787 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:07:40.787 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2023-09-22 13:07:14.538422390 +0000 2023-09-22T13:07:40.787 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2023-09-22 12:16:16.257118724 +0000 2023-09-22T13:07:40.787 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2023-09-22 12:16:16.257118724 +0000 2023-09-22T13:07:40.787 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2023-09-22T13:07:40.787 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-22T13:07:40.853 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2023-09-22T13:07:40.854 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2023-09-22T13:07:40.854 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000199388 s, 2.6 MB/s 2023-09-22T13:07:40.855 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-22T13:07:40.915 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_4 2023-09-22T13:07:40.974 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-22T13:07:40.974 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-22T13:07:40.974 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 6h/6d Inode: 426110 Links: 1 2023-09-22T13:07:40.974 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-22T13:07:40.975 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2023-09-22T13:07:40.975 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2023-09-22 13:07:14.538422390 +0000 2023-09-22T13:07:40.975 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2023-09-22 12:16:16.773108143 +0000 2023-09-22T13:07:40.975 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2023-09-22 12:16:16.773108143 +0000 2023-09-22T13:07:40.975 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2023-09-22T13:07:40.975 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-22T13:07:41.040 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2023-09-22T13:07:41.041 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2023-09-22T13:07:41.041 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000209146 s, 2.4 MB/s 2023-09-22T13:07:41.042 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-22T13:07:41.100 INFO:tasks.ceph:osd dev map: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:07:41.100 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi098.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi169.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2023-09-22T13:07:41.101 INFO:tasks.ceph:Generating config... 2023-09-22T13:07:41.102 INFO:tasks.ceph:[client] debug rgw = 20 2023-09-22T13:07:41.102 INFO:tasks.ceph:[client] rgw crypt require ssl = False 2023-09-22T13:07:41.102 INFO:tasks.ceph:[client] rgw crypt s3 kms backend = testing 2023-09-22T13:07:41.102 INFO:tasks.ceph:[client] rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= 2023-09-22T13:07:41.102 INFO:tasks.ceph:[client] rgw lc debug interval = 10 2023-09-22T13:07:41.103 INFO:tasks.ceph:[client] rgw max chunk size = 4194304 2023-09-22T13:07:41.103 INFO:tasks.ceph:[client] rgw obj stripe size = 6291456 2023-09-22T13:07:41.103 INFO:tasks.ceph:[client] setgroup = ceph 2023-09-22T13:07:41.103 INFO:tasks.ceph:[client] setuser = ceph 2023-09-22T13:07:41.103 INFO:tasks.ceph:[global] mon client directed command retry = 5 2023-09-22T13:07:41.103 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2023-09-22T13:07:41.103 INFO:tasks.ceph:[global] osd_max_pg_log_entries = 10 2023-09-22T13:07:41.103 INFO:tasks.ceph:[global] osd_min_pg_log_entries = 10 2023-09-22T13:07:41.104 INFO:tasks.ceph:[global] rgw override bucket index max shards = 1 2023-09-22T13:07:41.104 INFO:tasks.ceph:[mgr] debug mgr = 20 2023-09-22T13:07:41.104 INFO:tasks.ceph:[mgr] debug ms = 1 2023-09-22T13:07:41.104 INFO:tasks.ceph:[mon] debug mon = 20 2023-09-22T13:07:41.104 INFO:tasks.ceph:[mon] debug ms = 1 2023-09-22T13:07:41.104 INFO:tasks.ceph:[mon] debug paxos = 20 2023-09-22T13:07:41.104 INFO:tasks.ceph:[mon] lockdep = True 2023-09-22T13:07:41.104 INFO:tasks.ceph:[osd] debug ms = 1 2023-09-22T13:07:41.104 INFO:tasks.ceph:[osd] debug osd = 20 2023-09-22T13:07:41.104 INFO:tasks.ceph:[osd] lockdep = True 2023-09-22T13:07:41.105 INFO:tasks.ceph:[osd] osd objectstore = filestore 2023-09-22T13:07:41.105 INFO:tasks.ceph:[osd] osd shutdown pgref assert = True 2023-09-22T13:07:41.105 INFO:tasks.ceph:[osd] osd sloppy crc = True 2023-09-22T13:07:41.105 INFO:tasks.ceph:Setting up mon.a... 2023-09-22T13:07:41.105 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring 2023-09-22T13:07:41.219 INFO:teuthology.orchestra.run.smithi098.stdout:creating /etc/ceph/ceph.keyring 2023-09-22T13:07:41.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring 2023-09-22T13:07:41.266 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 0644 /etc/ceph/ceph.keyring 2023-09-22T13:07:41.336 DEBUG:tasks.ceph:Ceph mon addresses: [('mon.a', '172.21.15.98'), ('mon.c', '[v2:172.21.15.98:3301,v1:172.21.15.98:6790]'), ('mon.b', '172.21.15.169')] 2023-09-22T13:07:41.337 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.98,[v2:172.21.15.98:3301,v1:172.21.15.98:6790],172.21.15.169', 'mon client directed command retry': 5, 'ms inject socket failures': 5000, 'osd_max_pg_log_entries': 10, 'osd_min_pg_log_entries': 10, 'rgw override bucket index max shards': 1}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'debug ms': 1, 'debug osd': 20, 'lockdep': True, 'osd objectstore': 'filestore', 'osd shutdown pgref assert': True, 'osd sloppy crc': True}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'lockdep': True}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=', 'rgw lc debug interval': 10, 'rgw max chunk size': 4194304, 'rgw obj stripe size': 6291456, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mon.a': {}, 'mon.c': {}, 'mon.b': {}} 2023-09-22T13:07:41.337 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:41.338 DEBUG:teuthology.orchestra.run.smithi098:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2023-09-22T13:07:41.396 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.98 --addv c '[v2:172.21.15.98:3301,v1:172.21.15.98:6790]' --add b 172.21.15.169 --print /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:41.480 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:07:41.480 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:07:41.482 INFO:teuthology.orchestra.run.smithi098.stdout:monmaptool: monmap file /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:41.482 INFO:teuthology.orchestra.run.smithi098.stdout:monmaptool: generated fsid 583b11f8-631b-40f6-ac70-b90b0e6977fb 2023-09-22T13:07:41.482 INFO:teuthology.orchestra.run.smithi098.stdout:epoch 0 2023-09-22T13:07:41.483 INFO:teuthology.orchestra.run.smithi098.stdout:fsid 583b11f8-631b-40f6-ac70-b90b0e6977fb 2023-09-22T13:07:41.483 INFO:teuthology.orchestra.run.smithi098.stdout:last_changed 2023-09-22T13:07:41.481681+0000 2023-09-22T13:07:41.483 INFO:teuthology.orchestra.run.smithi098.stdout:created 2023-09-22T13:07:41.481681+0000 2023-09-22T13:07:41.483 INFO:teuthology.orchestra.run.smithi098.stdout:min_mon_release 0 (unknown) 2023-09-22T13:07:41.483 INFO:teuthology.orchestra.run.smithi098.stdout:election_strategy: 1 2023-09-22T13:07:41.483 INFO:teuthology.orchestra.run.smithi098.stdout:0: [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] mon.a 2023-09-22T13:07:41.484 INFO:teuthology.orchestra.run.smithi098.stdout:1: [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] mon.b 2023-09-22T13:07:41.484 INFO:teuthology.orchestra.run.smithi098.stdout:2: [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] mon.c 2023-09-22T13:07:41.484 INFO:teuthology.orchestra.run.smithi098.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/ceph.monmap (3 monitors) 2023-09-22T13:07:41.485 DEBUG:teuthology.orchestra.run.smithi098:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2023-09-22T13:07:41.503 INFO:tasks.ceph:Writing /etc/ceph/ceph.conf for FSID 583b11f8-631b-40f6-ac70-b90b0e6977fb... 2023-09-22T13:07:41.504 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2023-09-22T13:07:41.545 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2023-09-22T13:07:41.587 INFO:teuthology.orchestra.run.smithi098.stdout:[global] 2023-09-22T13:07:41.587 INFO:teuthology.orchestra.run.smithi098.stdout: chdir = "" 2023-09-22T13:07:41.588 INFO:teuthology.orchestra.run.smithi098.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2023-09-22T13:07:41.588 INFO:teuthology.orchestra.run.smithi098.stdout: auth supported = cephx 2023-09-22T13:07:41.588 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.588 INFO:teuthology.orchestra.run.smithi098.stdout: filestore xattr use omap = true 2023-09-22T13:07:41.588 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.588 INFO:teuthology.orchestra.run.smithi098.stdout: mon clock drift allowed = 1.000 2023-09-22T13:07:41.589 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.589 INFO:teuthology.orchestra.run.smithi098.stdout: osd crush chooseleaf type = 0 2023-09-22T13:07:41.589 INFO:teuthology.orchestra.run.smithi098.stdout: auth debug = true 2023-09-22T13:07:41.589 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.589 INFO:teuthology.orchestra.run.smithi098.stdout: ms die on old message = true 2023-09-22T13:07:41.589 INFO:teuthology.orchestra.run.smithi098.stdout: ms die on bug = true 2023-09-22T13:07:41.589 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.589 INFO:teuthology.orchestra.run.smithi098.stdout: mon max pg per osd = 10000 # >= luminous 2023-09-22T13:07:41.590 INFO:teuthology.orchestra.run.smithi098.stdout: mon pg warn max object skew = 0 2023-09-22T13:07:41.590 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.590 INFO:teuthology.orchestra.run.smithi098.stdout: # disable pg_autoscaler by default for new pools 2023-09-22T13:07:41.590 INFO:teuthology.orchestra.run.smithi098.stdout: osd_pool_default_pg_autoscale_mode = off 2023-09-22T13:07:41.590 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.590 INFO:teuthology.orchestra.run.smithi098.stdout: osd pool default size = 2 2023-09-22T13:07:41.590 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.591 INFO:teuthology.orchestra.run.smithi098.stdout: mon osd allow primary affinity = true 2023-09-22T13:07:41.591 INFO:teuthology.orchestra.run.smithi098.stdout: mon osd allow pg remap = true 2023-09-22T13:07:41.591 INFO:teuthology.orchestra.run.smithi098.stdout: mon warn on legacy crush tunables = false 2023-09-22T13:07:41.591 INFO:teuthology.orchestra.run.smithi098.stdout: mon warn on crush straw calc version zero = false 2023-09-22T13:07:41.591 INFO:teuthology.orchestra.run.smithi098.stdout: mon warn on no sortbitwise = false 2023-09-22T13:07:41.591 INFO:teuthology.orchestra.run.smithi098.stdout: mon warn on osd down out interval zero = false 2023-09-22T13:07:41.591 INFO:teuthology.orchestra.run.smithi098.stdout: mon warn on too few osds = false 2023-09-22T13:07:41.591 INFO:teuthology.orchestra.run.smithi098.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2023-09-22T13:07:41.592 INFO:teuthology.orchestra.run.smithi098.stdout: mon_warn_on_pool_no_redundancy = false 2023-09-22T13:07:41.592 INFO:teuthology.orchestra.run.smithi098.stdout: mon_allow_pool_size_one = true 2023-09-22T13:07:41.592 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.592 INFO:teuthology.orchestra.run.smithi098.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd 2023-09-22T13:07:41.592 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.592 INFO:teuthology.orchestra.run.smithi098.stdout: osd default data pool replay window = 5 2023-09-22T13:07:41.592 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.592 INFO:teuthology.orchestra.run.smithi098.stdout: mon allow pool delete = true 2023-09-22T13:07:41.592 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.593 INFO:teuthology.orchestra.run.smithi098.stdout: mon cluster log file level = debug 2023-09-22T13:07:41.593 INFO:teuthology.orchestra.run.smithi098.stdout: debug asserts on shutdown = true 2023-09-22T13:07:41.593 INFO:teuthology.orchestra.run.smithi098.stdout: mon health detail to clog = false 2023-09-22T13:07:41.593 INFO:teuthology.orchestra.run.smithi098.stdout: mon host = "172.21.15.98,[v2:172.21.15.98:3301,v1:172.21.15.98:6790],172.21.15.169" 2023-09-22T13:07:41.593 INFO:teuthology.orchestra.run.smithi098.stdout: mon client directed command retry = 5 2023-09-22T13:07:41.593 INFO:teuthology.orchestra.run.smithi098.stdout: ms inject socket failures = 5000 2023-09-22T13:07:41.593 INFO:teuthology.orchestra.run.smithi098.stdout: osd_max_pg_log_entries = 10 2023-09-22T13:07:41.593 INFO:teuthology.orchestra.run.smithi098.stdout: osd_min_pg_log_entries = 10 2023-09-22T13:07:41.593 INFO:teuthology.orchestra.run.smithi098.stdout: rgw override bucket index max shards = 1 2023-09-22T13:07:41.594 INFO:teuthology.orchestra.run.smithi098.stdout: fsid = 583b11f8-631b-40f6-ac70-b90b0e6977fb 2023-09-22T13:07:41.594 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.594 INFO:teuthology.orchestra.run.smithi098.stdout:[osd] 2023-09-22T13:07:41.594 INFO:teuthology.orchestra.run.smithi098.stdout: osd journal size = 100 2023-09-22T13:07:41.594 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.594 INFO:teuthology.orchestra.run.smithi098.stdout: osd scrub load threshold = 5.0 2023-09-22T13:07:41.594 INFO:teuthology.orchestra.run.smithi098.stdout: osd scrub max interval = 600 2023-09-22T13:07:41.594 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.595 INFO:teuthology.orchestra.run.smithi098.stdout: osd recover clone overlap = true 2023-09-22T13:07:41.595 INFO:teuthology.orchestra.run.smithi098.stdout: osd recovery max chunk = 1048576 2023-09-22T13:07:41.595 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.595 INFO:teuthology.orchestra.run.smithi098.stdout: osd debug shutdown = true 2023-09-22T13:07:41.595 INFO:teuthology.orchestra.run.smithi098.stdout: osd debug op order = true 2023-09-22T13:07:41.595 INFO:teuthology.orchestra.run.smithi098.stdout: osd debug verify stray on activate = true 2023-09-22T13:07:41.595 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.595 INFO:teuthology.orchestra.run.smithi098.stdout: osd open classes on start = true 2023-09-22T13:07:41.596 INFO:teuthology.orchestra.run.smithi098.stdout: osd debug pg log writeout = true 2023-09-22T13:07:41.596 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.596 INFO:teuthology.orchestra.run.smithi098.stdout: osd deep scrub update digest min age = 30 2023-09-22T13:07:41.596 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.596 INFO:teuthology.orchestra.run.smithi098.stdout: osd map max advance = 10 2023-09-22T13:07:41.596 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.596 INFO:teuthology.orchestra.run.smithi098.stdout: journal zero on create = true 2023-09-22T13:07:41.596 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.597 INFO:teuthology.orchestra.run.smithi098.stdout: filestore ondisk finisher threads = 3 2023-09-22T13:07:41.597 INFO:teuthology.orchestra.run.smithi098.stdout: filestore apply finisher threads = 3 2023-09-22T13:07:41.597 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.597 INFO:teuthology.orchestra.run.smithi098.stdout: bdev debug aio = true 2023-09-22T13:07:41.597 INFO:teuthology.orchestra.run.smithi098.stdout: osd debug misdirected ops = true 2023-09-22T13:07:41.597 INFO:teuthology.orchestra.run.smithi098.stdout: debug ms = 1 2023-09-22T13:07:41.597 INFO:teuthology.orchestra.run.smithi098.stdout: debug osd = 20 2023-09-22T13:07:41.597 INFO:teuthology.orchestra.run.smithi098.stdout: lockdep = True 2023-09-22T13:07:41.597 INFO:teuthology.orchestra.run.smithi098.stdout: osd objectstore = filestore 2023-09-22T13:07:41.598 INFO:teuthology.orchestra.run.smithi098.stdout: osd shutdown pgref assert = True 2023-09-22T13:07:41.598 INFO:teuthology.orchestra.run.smithi098.stdout: osd sloppy crc = True 2023-09-22T13:07:41.598 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.598 INFO:teuthology.orchestra.run.smithi098.stdout:[mgr] 2023-09-22T13:07:41.598 INFO:teuthology.orchestra.run.smithi098.stdout: debug ms = 1 2023-09-22T13:07:41.598 INFO:teuthology.orchestra.run.smithi098.stdout: debug mgr = 20 2023-09-22T13:07:41.598 INFO:teuthology.orchestra.run.smithi098.stdout: debug mon = 20 2023-09-22T13:07:41.598 INFO:teuthology.orchestra.run.smithi098.stdout: debug auth = 20 2023-09-22T13:07:41.599 INFO:teuthology.orchestra.run.smithi098.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:07:41.599 INFO:teuthology.orchestra.run.smithi098.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:07:41.599 INFO:teuthology.orchestra.run.smithi098.stdout: mgr/telemetry/nag = false 2023-09-22T13:07:41.599 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.599 INFO:teuthology.orchestra.run.smithi098.stdout:[mon] 2023-09-22T13:07:41.599 INFO:teuthology.orchestra.run.smithi098.stdout: debug ms = 1 2023-09-22T13:07:41.599 INFO:teuthology.orchestra.run.smithi098.stdout: debug mon = 20 2023-09-22T13:07:41.599 INFO:teuthology.orchestra.run.smithi098.stdout: debug paxos = 20 2023-09-22T13:07:41.599 INFO:teuthology.orchestra.run.smithi098.stdout: debug auth = 20 2023-09-22T13:07:41.600 INFO:teuthology.orchestra.run.smithi098.stdout: mon data avail warn = 5 2023-09-22T13:07:41.600 INFO:teuthology.orchestra.run.smithi098.stdout: mon mgr mkfs grace = 240 2023-09-22T13:07:41.600 INFO:teuthology.orchestra.run.smithi098.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:07:41.600 INFO:teuthology.orchestra.run.smithi098.stdout: mon osd reporter subtree level = osd 2023-09-22T13:07:41.600 INFO:teuthology.orchestra.run.smithi098.stdout: mon osd prime pg temp = true 2023-09-22T13:07:41.600 INFO:teuthology.orchestra.run.smithi098.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:07:41.600 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.600 INFO:teuthology.orchestra.run.smithi098.stdout: # rotate auth tickets quickly to exercise renewal paths 2023-09-22T13:07:41.600 INFO:teuthology.orchestra.run.smithi098.stdout: auth mon ticket ttl = 660 # 11m 2023-09-22T13:07:41.601 INFO:teuthology.orchestra.run.smithi098.stdout: auth service ticket ttl = 240 # 4m 2023-09-22T13:07:41.601 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.601 INFO:teuthology.orchestra.run.smithi098.stdout: # don't complain about insecure global_id in the test suite 2023-09-22T13:07:41.601 INFO:teuthology.orchestra.run.smithi098.stdout: mon_warn_on_insecure_global_id_reclaim = false 2023-09-22T13:07:41.601 INFO:teuthology.orchestra.run.smithi098.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2023-09-22T13:07:41.601 INFO:teuthology.orchestra.run.smithi098.stdout: lockdep = True 2023-09-22T13:07:41.601 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:07:41.601 INFO:teuthology.orchestra.run.smithi098.stdout:[client] 2023-09-22T13:07:41.601 INFO:teuthology.orchestra.run.smithi098.stdout: rgw cache enabled = true 2023-09-22T13:07:41.602 INFO:teuthology.orchestra.run.smithi098.stdout: rgw enable ops log = true 2023-09-22T13:07:41.602 INFO:teuthology.orchestra.run.smithi098.stdout: rgw enable usage log = true 2023-09-22T13:07:41.602 INFO:teuthology.orchestra.run.smithi098.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2023-09-22T13:07:41.602 INFO:teuthology.orchestra.run.smithi098.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2023-09-22T13:07:41.602 INFO:teuthology.orchestra.run.smithi098.stdout: debug rgw = 20 2023-09-22T13:07:41.602 INFO:teuthology.orchestra.run.smithi098.stdout: rgw crypt require ssl = False 2023-09-22T13:07:41.602 INFO:teuthology.orchestra.run.smithi098.stdout: rgw crypt s3 kms backend = testing 2023-09-22T13:07:41.602 INFO:teuthology.orchestra.run.smithi098.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= 2023-09-22T13:07:41.602 INFO:teuthology.orchestra.run.smithi098.stdout: rgw lc debug interval = 10 2023-09-22T13:07:41.603 INFO:teuthology.orchestra.run.smithi098.stdout: rgw max chunk size = 4194304 2023-09-22T13:07:41.603 INFO:teuthology.orchestra.run.smithi098.stdout: rgw obj stripe size = 6291456 2023-09-22T13:07:41.603 INFO:teuthology.orchestra.run.smithi098.stdout: setgroup = ceph 2023-09-22T13:07:41.603 INFO:teuthology.orchestra.run.smithi098.stdout: setuser = ceph 2023-09-22T13:07:41.603 INFO:teuthology.orchestra.run.smithi098.stdout:[mon.a] 2023-09-22T13:07:41.603 INFO:teuthology.orchestra.run.smithi098.stdout:[mon.c] 2023-09-22T13:07:41.603 INFO:teuthology.orchestra.run.smithi098.stdout:[mon.b] 2023-09-22T13:07:41.605 INFO:teuthology.orchestra.run.smithi169.stdout:[global] 2023-09-22T13:07:41.605 INFO:teuthology.orchestra.run.smithi169.stdout: chdir = "" 2023-09-22T13:07:41.605 INFO:teuthology.orchestra.run.smithi169.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2023-09-22T13:07:41.605 INFO:teuthology.orchestra.run.smithi169.stdout: auth supported = cephx 2023-09-22T13:07:41.606 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.606 INFO:teuthology.orchestra.run.smithi169.stdout: filestore xattr use omap = true 2023-09-22T13:07:41.606 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.606 INFO:teuthology.orchestra.run.smithi169.stdout: mon clock drift allowed = 1.000 2023-09-22T13:07:41.606 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.606 INFO:teuthology.orchestra.run.smithi169.stdout: osd crush chooseleaf type = 0 2023-09-22T13:07:41.606 INFO:teuthology.orchestra.run.smithi169.stdout: auth debug = true 2023-09-22T13:07:41.606 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.607 INFO:teuthology.orchestra.run.smithi169.stdout: ms die on old message = true 2023-09-22T13:07:41.607 INFO:teuthology.orchestra.run.smithi169.stdout: ms die on bug = true 2023-09-22T13:07:41.607 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.607 INFO:teuthology.orchestra.run.smithi169.stdout: mon max pg per osd = 10000 # >= luminous 2023-09-22T13:07:41.607 INFO:teuthology.orchestra.run.smithi169.stdout: mon pg warn max object skew = 0 2023-09-22T13:07:41.607 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.607 INFO:teuthology.orchestra.run.smithi169.stdout: # disable pg_autoscaler by default for new pools 2023-09-22T13:07:41.607 INFO:teuthology.orchestra.run.smithi169.stdout: osd_pool_default_pg_autoscale_mode = off 2023-09-22T13:07:41.607 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.607 INFO:teuthology.orchestra.run.smithi169.stdout: osd pool default size = 2 2023-09-22T13:07:41.608 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.608 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd allow primary affinity = true 2023-09-22T13:07:41.608 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd allow pg remap = true 2023-09-22T13:07:41.608 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on legacy crush tunables = false 2023-09-22T13:07:41.608 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on crush straw calc version zero = false 2023-09-22T13:07:41.608 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on no sortbitwise = false 2023-09-22T13:07:41.608 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on osd down out interval zero = false 2023-09-22T13:07:41.608 INFO:teuthology.orchestra.run.smithi169.stdout: mon warn on too few osds = false 2023-09-22T13:07:41.608 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2023-09-22T13:07:41.609 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_pool_no_redundancy = false 2023-09-22T13:07:41.609 INFO:teuthology.orchestra.run.smithi169.stdout: mon_allow_pool_size_one = true 2023-09-22T13:07:41.609 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.609 INFO:teuthology.orchestra.run.smithi169.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd 2023-09-22T13:07:41.609 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.609 INFO:teuthology.orchestra.run.smithi169.stdout: osd default data pool replay window = 5 2023-09-22T13:07:41.609 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.609 INFO:teuthology.orchestra.run.smithi169.stdout: mon allow pool delete = true 2023-09-22T13:07:41.609 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.610 INFO:teuthology.orchestra.run.smithi169.stdout: mon cluster log file level = debug 2023-09-22T13:07:41.610 INFO:teuthology.orchestra.run.smithi169.stdout: debug asserts on shutdown = true 2023-09-22T13:07:41.610 INFO:teuthology.orchestra.run.smithi169.stdout: mon health detail to clog = false 2023-09-22T13:07:41.610 INFO:teuthology.orchestra.run.smithi169.stdout: mon host = "172.21.15.98,[v2:172.21.15.98:3301,v1:172.21.15.98:6790],172.21.15.169" 2023-09-22T13:07:41.610 INFO:teuthology.orchestra.run.smithi169.stdout: mon client directed command retry = 5 2023-09-22T13:07:41.610 INFO:teuthology.orchestra.run.smithi169.stdout: ms inject socket failures = 5000 2023-09-22T13:07:41.610 INFO:teuthology.orchestra.run.smithi169.stdout: osd_max_pg_log_entries = 10 2023-09-22T13:07:41.610 INFO:teuthology.orchestra.run.smithi169.stdout: osd_min_pg_log_entries = 10 2023-09-22T13:07:41.610 INFO:teuthology.orchestra.run.smithi169.stdout: rgw override bucket index max shards = 1 2023-09-22T13:07:41.611 INFO:teuthology.orchestra.run.smithi169.stdout: fsid = 583b11f8-631b-40f6-ac70-b90b0e6977fb 2023-09-22T13:07:41.611 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.611 INFO:teuthology.orchestra.run.smithi169.stdout:[osd] 2023-09-22T13:07:41.611 INFO:teuthology.orchestra.run.smithi169.stdout: osd journal size = 100 2023-09-22T13:07:41.611 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.611 INFO:teuthology.orchestra.run.smithi169.stdout: osd scrub load threshold = 5.0 2023-09-22T13:07:41.611 INFO:teuthology.orchestra.run.smithi169.stdout: osd scrub max interval = 600 2023-09-22T13:07:41.611 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.611 INFO:teuthology.orchestra.run.smithi169.stdout: osd recover clone overlap = true 2023-09-22T13:07:41.611 INFO:teuthology.orchestra.run.smithi169.stdout: osd recovery max chunk = 1048576 2023-09-22T13:07:41.612 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.612 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug shutdown = true 2023-09-22T13:07:41.612 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug op order = true 2023-09-22T13:07:41.612 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug verify stray on activate = true 2023-09-22T13:07:41.612 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.612 INFO:teuthology.orchestra.run.smithi169.stdout: osd open classes on start = true 2023-09-22T13:07:41.612 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug pg log writeout = true 2023-09-22T13:07:41.612 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.612 INFO:teuthology.orchestra.run.smithi169.stdout: osd deep scrub update digest min age = 30 2023-09-22T13:07:41.613 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.613 INFO:teuthology.orchestra.run.smithi169.stdout: osd map max advance = 10 2023-09-22T13:07:41.613 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.613 INFO:teuthology.orchestra.run.smithi169.stdout: journal zero on create = true 2023-09-22T13:07:41.613 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.613 INFO:teuthology.orchestra.run.smithi169.stdout: filestore ondisk finisher threads = 3 2023-09-22T13:07:41.613 INFO:teuthology.orchestra.run.smithi169.stdout: filestore apply finisher threads = 3 2023-09-22T13:07:41.613 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.613 INFO:teuthology.orchestra.run.smithi169.stdout: bdev debug aio = true 2023-09-22T13:07:41.613 INFO:teuthology.orchestra.run.smithi169.stdout: osd debug misdirected ops = true 2023-09-22T13:07:41.614 INFO:teuthology.orchestra.run.smithi169.stdout: debug ms = 1 2023-09-22T13:07:41.614 INFO:teuthology.orchestra.run.smithi169.stdout: debug osd = 20 2023-09-22T13:07:41.614 INFO:teuthology.orchestra.run.smithi169.stdout: lockdep = True 2023-09-22T13:07:41.614 INFO:teuthology.orchestra.run.smithi169.stdout: osd objectstore = filestore 2023-09-22T13:07:41.614 INFO:teuthology.orchestra.run.smithi169.stdout: osd shutdown pgref assert = True 2023-09-22T13:07:41.614 INFO:teuthology.orchestra.run.smithi169.stdout: osd sloppy crc = True 2023-09-22T13:07:41.614 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.614 INFO:teuthology.orchestra.run.smithi169.stdout:[mgr] 2023-09-22T13:07:41.614 INFO:teuthology.orchestra.run.smithi169.stdout: debug ms = 1 2023-09-22T13:07:41.615 INFO:teuthology.orchestra.run.smithi169.stdout: debug mgr = 20 2023-09-22T13:07:41.615 INFO:teuthology.orchestra.run.smithi169.stdout: debug mon = 20 2023-09-22T13:07:41.615 INFO:teuthology.orchestra.run.smithi169.stdout: debug auth = 20 2023-09-22T13:07:41.615 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:07:41.615 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:07:41.615 INFO:teuthology.orchestra.run.smithi169.stdout: mgr/telemetry/nag = false 2023-09-22T13:07:41.615 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.615 INFO:teuthology.orchestra.run.smithi169.stdout:[mon] 2023-09-22T13:07:41.615 INFO:teuthology.orchestra.run.smithi169.stdout: debug ms = 1 2023-09-22T13:07:41.616 INFO:teuthology.orchestra.run.smithi169.stdout: debug mon = 20 2023-09-22T13:07:41.616 INFO:teuthology.orchestra.run.smithi169.stdout: debug paxos = 20 2023-09-22T13:07:41.616 INFO:teuthology.orchestra.run.smithi169.stdout: debug auth = 20 2023-09-22T13:07:41.616 INFO:teuthology.orchestra.run.smithi169.stdout: mon data avail warn = 5 2023-09-22T13:07:41.616 INFO:teuthology.orchestra.run.smithi169.stdout: mon mgr mkfs grace = 240 2023-09-22T13:07:41.616 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min pgs per osd = 4 2023-09-22T13:07:41.616 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd reporter subtree level = osd 2023-09-22T13:07:41.616 INFO:teuthology.orchestra.run.smithi169.stdout: mon osd prime pg temp = true 2023-09-22T13:07:41.616 INFO:teuthology.orchestra.run.smithi169.stdout: mon reweight min bytes per osd = 10 2023-09-22T13:07:41.616 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.617 INFO:teuthology.orchestra.run.smithi169.stdout: # rotate auth tickets quickly to exercise renewal paths 2023-09-22T13:07:41.617 INFO:teuthology.orchestra.run.smithi169.stdout: auth mon ticket ttl = 660 # 11m 2023-09-22T13:07:41.617 INFO:teuthology.orchestra.run.smithi169.stdout: auth service ticket ttl = 240 # 4m 2023-09-22T13:07:41.617 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.617 INFO:teuthology.orchestra.run.smithi169.stdout: # don't complain about insecure global_id in the test suite 2023-09-22T13:07:41.617 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_insecure_global_id_reclaim = false 2023-09-22T13:07:41.617 INFO:teuthology.orchestra.run.smithi169.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2023-09-22T13:07:41.617 INFO:teuthology.orchestra.run.smithi169.stdout: lockdep = True 2023-09-22T13:07:41.617 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-09-22T13:07:41.618 INFO:teuthology.orchestra.run.smithi169.stdout:[client] 2023-09-22T13:07:41.618 INFO:teuthology.orchestra.run.smithi169.stdout: rgw cache enabled = true 2023-09-22T13:07:41.618 INFO:teuthology.orchestra.run.smithi169.stdout: rgw enable ops log = true 2023-09-22T13:07:41.618 INFO:teuthology.orchestra.run.smithi169.stdout: rgw enable usage log = true 2023-09-22T13:07:41.618 INFO:teuthology.orchestra.run.smithi169.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2023-09-22T13:07:41.618 INFO:teuthology.orchestra.run.smithi169.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2023-09-22T13:07:41.618 INFO:teuthology.orchestra.run.smithi169.stdout: debug rgw = 20 2023-09-22T13:07:41.618 INFO:teuthology.orchestra.run.smithi169.stdout: rgw crypt require ssl = False 2023-09-22T13:07:41.618 INFO:teuthology.orchestra.run.smithi169.stdout: rgw crypt s3 kms backend = testing 2023-09-22T13:07:41.619 INFO:teuthology.orchestra.run.smithi169.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= 2023-09-22T13:07:41.619 INFO:teuthology.orchestra.run.smithi169.stdout: rgw lc debug interval = 10 2023-09-22T13:07:41.619 INFO:teuthology.orchestra.run.smithi169.stdout: rgw max chunk size = 4194304 2023-09-22T13:07:41.619 INFO:teuthology.orchestra.run.smithi169.stdout: rgw obj stripe size = 6291456 2023-09-22T13:07:41.619 INFO:teuthology.orchestra.run.smithi169.stdout: setgroup = ceph 2023-09-22T13:07:41.619 INFO:teuthology.orchestra.run.smithi169.stdout: setuser = ceph 2023-09-22T13:07:41.619 INFO:teuthology.orchestra.run.smithi169.stdout:[mon.a] 2023-09-22T13:07:41.619 INFO:teuthology.orchestra.run.smithi169.stdout:[mon.c] 2023-09-22T13:07:41.619 INFO:teuthology.orchestra.run.smithi169.stdout:[mon.b] 2023-09-22T13:07:41.620 INFO:tasks.ceph:Creating admin key on mon.a... 2023-09-22T13:07:41.620 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/ceph.keyring 2023-09-22T13:07:41.698 INFO:tasks.ceph:Copying monmap to all nodes... 2023-09-22T13:07:41.698 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:41.698 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/etc/ceph/ceph.keyring of=/dev/stdout 2023-09-22T13:07:41.716 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:41.716 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/home/ubuntu/cephtest/ceph.monmap of=/dev/stdout 2023-09-22T13:07:41.775 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi098.front.sepia.ceph.com 2023-09-22T13:07:41.775 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:41.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/ceph/ceph.keyring 2023-09-22T13:07:41.775 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 0644 /etc/ceph/ceph.keyring 2023-09-22T13:07:41.851 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:41.851 DEBUG:teuthology.orchestra.run.smithi098:> dd of=/home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:41.910 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi169.front.sepia.ceph.com 2023-09-22T13:07:41.910 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:41.910 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/ceph/ceph.keyring 2023-09-22T13:07:41.910 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 0644 /etc/ceph/ceph.keyring 2023-09-22T13:07:41.948 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:41.949 DEBUG:teuthology.orchestra.run.smithi169:> dd of=/home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:42.007 INFO:tasks.ceph:Setting up mon nodes... 2023-09-22T13:07:42.007 INFO:tasks.ceph:Setting up mgr nodes... 2023-09-22T13:07:42.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/mgr/ceph-y && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.y /var/lib/ceph/mgr/ceph-y/keyring 2023-09-22T13:07:42.057 INFO:teuthology.orchestra.run.smithi098.stdout:creating /var/lib/ceph/mgr/ceph-y/keyring 2023-09-22T13:07:42.059 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /var/lib/ceph/mgr/ceph-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/ceph-x/keyring 2023-09-22T13:07:42.188 INFO:teuthology.orchestra.run.smithi169.stdout:creating /var/lib/ceph/mgr/ceph-x/keyring 2023-09-22T13:07:42.191 INFO:tasks.ceph:Setting up mds nodes... 2023-09-22T13:07:42.191 INFO:tasks.ceph_client:Setting up client nodes... 2023-09-22T13:07:42.191 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/ceph.client.0.keyring && sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-22T13:07:42.232 INFO:teuthology.orchestra.run.smithi098.stdout:creating /etc/ceph/ceph.client.0.keyring 2023-09-22T13:07:42.245 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/ceph.client.1.keyring && sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-22T13:07:42.285 INFO:teuthology.orchestra.run.smithi169.stdout:creating /etc/ceph/ceph.client.1.keyring 2023-09-22T13:07:42.297 INFO:tasks.ceph:Running mkfs on osd nodes... 2023-09-22T13:07:42.298 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi098.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}, Remote(name='ubuntu@smithi169.front.sepia.ceph.com'): {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'}} 2023-09-22T13:07:42.298 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/osd/ceph-0 2023-09-22T13:07:42.326 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:07:42.326 INFO:tasks.ceph:role: osd.0 2023-09-22T13:07:42.326 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi098.front.sepia.ceph.com 2023-09-22T13:07:42.327 DEBUG:teuthology.orchestra.run.smithi098:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2023-09-22T13:07:42.401 INFO:teuthology.orchestra.run.smithi098.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:07:42.401 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:07:42.401 INFO:teuthology.orchestra.run.smithi098.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:07:42.402 INFO:teuthology.orchestra.run.smithi098.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:07:42.402 INFO:teuthology.orchestra.run.smithi098.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:07:42.402 INFO:teuthology.orchestra.run.smithi098.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:07:42.402 INFO:teuthology.orchestra.run.smithi098.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:07:42.402 INFO:teuthology.orchestra.run.smithi098.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:07:42.402 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:07:42.402 INFO:teuthology.orchestra.run.smithi098.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:07:42.823 INFO:teuthology.orchestra.run.smithi098.stdout:Discarding blocks...Done. 2023-09-22T13:07:42.825 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi098.front.sepia.ceph.com -o noatime 2023-09-22T13:07:42.825 DEBUG:teuthology.orchestra.run.smithi098:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-0 2023-09-22T13:07:42.992 DEBUG:teuthology.orchestra.run.smithi098:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-0 2023-09-22T13:07:43.026 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/osd/ceph-1 2023-09-22T13:07:43.093 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:07:43.094 INFO:tasks.ceph:role: osd.1 2023-09-22T13:07:43.094 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi098.front.sepia.ceph.com 2023-09-22T13:07:43.094 DEBUG:teuthology.orchestra.run.smithi098:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2023-09-22T13:07:43.168 INFO:teuthology.orchestra.run.smithi098.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:07:43.168 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:07:43.168 INFO:teuthology.orchestra.run.smithi098.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:07:43.169 INFO:teuthology.orchestra.run.smithi098.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:07:43.169 INFO:teuthology.orchestra.run.smithi098.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:07:43.169 INFO:teuthology.orchestra.run.smithi098.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:07:43.169 INFO:teuthology.orchestra.run.smithi098.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:07:43.169 INFO:teuthology.orchestra.run.smithi098.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:07:43.169 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:07:43.169 INFO:teuthology.orchestra.run.smithi098.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:07:43.586 INFO:teuthology.orchestra.run.smithi098.stdout:Discarding blocks...Done. 2023-09-22T13:07:43.588 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi098.front.sepia.ceph.com -o noatime 2023-09-22T13:07:43.588 DEBUG:teuthology.orchestra.run.smithi098:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-1 2023-09-22T13:07:43.696 DEBUG:teuthology.orchestra.run.smithi098:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-1 2023-09-22T13:07:43.728 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/osd/ceph-2 2023-09-22T13:07:43.795 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:07:43.795 INFO:tasks.ceph:role: osd.2 2023-09-22T13:07:43.795 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi098.front.sepia.ceph.com 2023-09-22T13:07:43.795 DEBUG:teuthology.orchestra.run.smithi098:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2023-09-22T13:07:43.870 INFO:teuthology.orchestra.run.smithi098.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:07:43.870 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:07:43.871 INFO:teuthology.orchestra.run.smithi098.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:07:43.871 INFO:teuthology.orchestra.run.smithi098.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:07:43.871 INFO:teuthology.orchestra.run.smithi098.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:07:43.871 INFO:teuthology.orchestra.run.smithi098.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:07:43.871 INFO:teuthology.orchestra.run.smithi098.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:07:43.871 INFO:teuthology.orchestra.run.smithi098.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:07:43.872 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:07:43.872 INFO:teuthology.orchestra.run.smithi098.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:07:44.327 INFO:teuthology.orchestra.run.smithi098.stdout:Discarding blocks...Done. 2023-09-22T13:07:44.329 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi098.front.sepia.ceph.com -o noatime 2023-09-22T13:07:44.329 DEBUG:teuthology.orchestra.run.smithi098:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-2 2023-09-22T13:07:44.501 DEBUG:teuthology.orchestra.run.smithi098:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-2 2023-09-22T13:07:44.535 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/osd/ceph-3 2023-09-22T13:07:44.603 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2023-09-22T13:07:44.603 INFO:tasks.ceph:role: osd.3 2023-09-22T13:07:44.603 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi098.front.sepia.ceph.com 2023-09-22T13:07:44.604 DEBUG:teuthology.orchestra.run.smithi098:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2023-09-22T13:07:44.677 INFO:teuthology.orchestra.run.smithi098.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:07:44.678 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:07:44.678 INFO:teuthology.orchestra.run.smithi098.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:07:44.678 INFO:teuthology.orchestra.run.smithi098.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:07:44.678 INFO:teuthology.orchestra.run.smithi098.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:07:44.678 INFO:teuthology.orchestra.run.smithi098.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:07:44.678 INFO:teuthology.orchestra.run.smithi098.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:07:44.679 INFO:teuthology.orchestra.run.smithi098.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:07:44.679 INFO:teuthology.orchestra.run.smithi098.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:07:44.679 INFO:teuthology.orchestra.run.smithi098.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:07:45.106 INFO:teuthology.orchestra.run.smithi098.stdout:Discarding blocks...Done. 2023-09-22T13:07:45.107 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi098.front.sepia.ceph.com -o noatime 2023-09-22T13:07:45.108 DEBUG:teuthology.orchestra.run.smithi098:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-3 2023-09-22T13:07:45.292 DEBUG:teuthology.orchestra.run.smithi098:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-3 2023-09-22T13:07:45.327 DEBUG:teuthology.orchestra.run.smithi098:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:45.420 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.419+0000 7f80ba5d5200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2023-09-22T13:07:45.421 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.420+0000 7f80ba5d5200 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2023-09-22T13:07:45.429 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.428+0000 7f80ba5d5200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:45.429 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.428+0000 7f80ba5d5200 -1 journal FileJournal::_open_file : zeroing journal 2023-09-22T13:07:45.633 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.632+0000 7f80ba5d5200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:45.633 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.632+0000 7f80ba5d5200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:45.633 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.632+0000 7f80ba5d5200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:45.651 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-22T13:07:45.679 DEBUG:teuthology.orchestra.run.smithi098:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:45.763 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.762+0000 7f0231de0200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2023-09-22T13:07:45.763 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.762+0000 7f0231de0200 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2023-09-22T13:07:45.767 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.766+0000 7f0231de0200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:45.768 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.766+0000 7f0231de0200 -1 journal FileJournal::_open_file : zeroing journal 2023-09-22T13:07:45.904 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.903+0000 7f0231de0200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:45.905 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.903+0000 7f0231de0200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:45.905 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:45.903+0000 7f0231de0200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:45.912 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-22T13:07:45.980 DEBUG:teuthology.orchestra.run.smithi098:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:46.063 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.062+0000 7f06e4f61200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2023-09-22T13:07:46.064 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.062+0000 7f06e4f61200 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2023-09-22T13:07:46.068 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.067+0000 7f06e4f61200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:46.068 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.067+0000 7f06e4f61200 -1 journal FileJournal::_open_file : zeroing journal 2023-09-22T13:07:46.212 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.211+0000 7f06e4f61200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:46.213 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.212+0000 7f06e4f61200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:46.213 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.212+0000 7f06e4f61200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:46.222 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-22T13:07:46.289 DEBUG:teuthology.orchestra.run.smithi098:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:46.372 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.372+0000 7fc631f4e200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2023-09-22T13:07:46.372 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.372+0000 7fc631f4e200 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2023-09-22T13:07:46.379 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.378+0000 7fc631f4e200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:46.379 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.378+0000 7fc631f4e200 -1 journal FileJournal::_open_file : zeroing journal 2023-09-22T13:07:46.514 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.513+0000 7fc631f4e200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:46.514 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.514+0000 7fc631f4e200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:46.515 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:07:46.514+0000 7fc631f4e200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:46.524 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-22T13:07:46.592 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /var/lib/ceph/osd/ceph-4 2023-09-22T13:07:46.625 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:07:46.625 INFO:tasks.ceph:role: osd.4 2023-09-22T13:07:46.625 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi169.front.sepia.ceph.com 2023-09-22T13:07:46.626 DEBUG:teuthology.orchestra.run.smithi169:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2023-09-22T13:07:46.704 INFO:teuthology.orchestra.run.smithi169.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:07:46.704 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:07:46.704 INFO:teuthology.orchestra.run.smithi169.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:07:46.704 INFO:teuthology.orchestra.run.smithi169.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:07:46.704 INFO:teuthology.orchestra.run.smithi169.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:07:46.704 INFO:teuthology.orchestra.run.smithi169.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:07:46.704 INFO:teuthology.orchestra.run.smithi169.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:07:46.705 INFO:teuthology.orchestra.run.smithi169.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:07:46.705 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:07:46.705 INFO:teuthology.orchestra.run.smithi169.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:07:47.135 INFO:teuthology.orchestra.run.smithi169.stdout:Discarding blocks...Done. 2023-09-22T13:07:47.137 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi169.front.sepia.ceph.com -o noatime 2023-09-22T13:07:47.137 DEBUG:teuthology.orchestra.run.smithi169:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-4 2023-09-22T13:07:47.325 DEBUG:teuthology.orchestra.run.smithi169:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-4 2023-09-22T13:07:47.363 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /var/lib/ceph/osd/ceph-5 2023-09-22T13:07:47.433 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:07:47.433 INFO:tasks.ceph:role: osd.5 2023-09-22T13:07:47.433 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi169.front.sepia.ceph.com 2023-09-22T13:07:47.433 DEBUG:teuthology.orchestra.run.smithi169:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2023-09-22T13:07:47.509 INFO:teuthology.orchestra.run.smithi169.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:07:47.509 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:07:47.509 INFO:teuthology.orchestra.run.smithi169.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:07:47.510 INFO:teuthology.orchestra.run.smithi169.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:07:47.510 INFO:teuthology.orchestra.run.smithi169.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:07:47.510 INFO:teuthology.orchestra.run.smithi169.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:07:47.510 INFO:teuthology.orchestra.run.smithi169.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:07:47.510 INFO:teuthology.orchestra.run.smithi169.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:07:47.510 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:07:47.510 INFO:teuthology.orchestra.run.smithi169.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:07:47.934 INFO:teuthology.orchestra.run.smithi169.stdout:Discarding blocks...Done. 2023-09-22T13:07:47.937 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi169.front.sepia.ceph.com -o noatime 2023-09-22T13:07:47.937 DEBUG:teuthology.orchestra.run.smithi169:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-5 2023-09-22T13:07:48.052 DEBUG:teuthology.orchestra.run.smithi169:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-5 2023-09-22T13:07:48.087 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /var/lib/ceph/osd/ceph-6 2023-09-22T13:07:48.158 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:07:48.158 INFO:tasks.ceph:role: osd.6 2023-09-22T13:07:48.158 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi169.front.sepia.ceph.com 2023-09-22T13:07:48.158 DEBUG:teuthology.orchestra.run.smithi169:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2023-09-22T13:07:48.235 INFO:teuthology.orchestra.run.smithi169.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:07:48.235 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:07:48.235 INFO:teuthology.orchestra.run.smithi169.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:07:48.235 INFO:teuthology.orchestra.run.smithi169.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:07:48.235 INFO:teuthology.orchestra.run.smithi169.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:07:48.235 INFO:teuthology.orchestra.run.smithi169.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:07:48.235 INFO:teuthology.orchestra.run.smithi169.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:07:48.235 INFO:teuthology.orchestra.run.smithi169.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:07:48.236 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:07:48.236 INFO:teuthology.orchestra.run.smithi169.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:07:48.680 INFO:teuthology.orchestra.run.smithi169.stdout:Discarding blocks...Done. 2023-09-22T13:07:48.682 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi169.front.sepia.ceph.com -o noatime 2023-09-22T13:07:48.682 DEBUG:teuthology.orchestra.run.smithi169:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-6 2023-09-22T13:07:48.845 DEBUG:teuthology.orchestra.run.smithi169:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-6 2023-09-22T13:07:48.883 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /var/lib/ceph/osd/ceph-7 2023-09-22T13:07:48.954 INFO:tasks.ceph:roles_to_devs: {'osd.4': '/dev/vg_nvme/lv_1', 'osd.5': '/dev/vg_nvme/lv_2', 'osd.6': '/dev/vg_nvme/lv_3', 'osd.7': '/dev/vg_nvme/lv_4'} 2023-09-22T13:07:48.954 INFO:tasks.ceph:role: osd.7 2023-09-22T13:07:48.954 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi169.front.sepia.ceph.com 2023-09-22T13:07:48.954 DEBUG:teuthology.orchestra.run.smithi169:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2023-09-22T13:07:49.031 INFO:teuthology.orchestra.run.smithi169.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2023-09-22T13:07:49.032 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 attr=2, projid32bit=1 2023-09-22T13:07:49.032 INFO:teuthology.orchestra.run.smithi169.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2023-09-22T13:07:49.032 INFO:teuthology.orchestra.run.smithi169.stdout: = reflink=1 bigtime=0 inobtcount=0 2023-09-22T13:07:49.032 INFO:teuthology.orchestra.run.smithi169.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2023-09-22T13:07:49.032 INFO:teuthology.orchestra.run.smithi169.stdout: = sunit=0 swidth=0 blks 2023-09-22T13:07:49.032 INFO:teuthology.orchestra.run.smithi169.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2023-09-22T13:07:49.032 INFO:teuthology.orchestra.run.smithi169.stdout:log =internal log bsize=4096 blocks=11443, version=2 2023-09-22T13:07:49.032 INFO:teuthology.orchestra.run.smithi169.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2023-09-22T13:07:49.032 INFO:teuthology.orchestra.run.smithi169.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2023-09-22T13:07:49.470 INFO:teuthology.orchestra.run.smithi169.stdout:Discarding blocks...Done. 2023-09-22T13:07:49.472 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi169.front.sepia.ceph.com -o noatime 2023-09-22T13:07:49.472 DEBUG:teuthology.orchestra.run.smithi169:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-7 2023-09-22T13:07:49.643 DEBUG:teuthology.orchestra.run.smithi169:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-7 2023-09-22T13:07:49.685 DEBUG:teuthology.orchestra.run.smithi169:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 4 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:49.779 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:49.778+0000 7fae15e93200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-4/keyring: can't open /var/lib/ceph/osd/ceph-4/keyring: (2) No such file or directory 2023-09-22T13:07:49.779 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:49.778+0000 7fae15e93200 -1 created new key in keyring /var/lib/ceph/osd/ceph-4/keyring 2023-09-22T13:07:49.787 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:49.787+0000 7fae15e93200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:49.788 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:49.787+0000 7fae15e93200 -1 journal FileJournal::_open_file : zeroing journal 2023-09-22T13:07:49.997 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:49.996+0000 7fae15e93200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:49.997 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:49.996+0000 7fae15e93200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:49.997 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:49.996+0000 7fae15e93200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:50.014 DEBUG:teuthology.orchestra.run.smithi169:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-09-22T13:07:50.043 DEBUG:teuthology.orchestra.run.smithi169:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 5 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:50.128 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.127+0000 7ff06ebcf200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-5/keyring: can't open /var/lib/ceph/osd/ceph-5/keyring: (2) No such file or directory 2023-09-22T13:07:50.128 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.127+0000 7ff06ebcf200 -1 created new key in keyring /var/lib/ceph/osd/ceph-5/keyring 2023-09-22T13:07:50.133 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.132+0000 7ff06ebcf200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:50.133 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.132+0000 7ff06ebcf200 -1 journal FileJournal::_open_file : zeroing journal 2023-09-22T13:07:50.287 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.286+0000 7ff06ebcf200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:50.287 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.286+0000 7ff06ebcf200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:50.287 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.286+0000 7ff06ebcf200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:50.295 DEBUG:teuthology.orchestra.run.smithi169:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-09-22T13:07:50.363 DEBUG:teuthology.orchestra.run.smithi169:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 6 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:50.446 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.445+0000 7f1409beb200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-6/keyring: can't open /var/lib/ceph/osd/ceph-6/keyring: (2) No such file or directory 2023-09-22T13:07:50.447 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.446+0000 7f1409beb200 -1 created new key in keyring /var/lib/ceph/osd/ceph-6/keyring 2023-09-22T13:07:50.451 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.450+0000 7f1409beb200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:50.451 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.450+0000 7f1409beb200 -1 journal FileJournal::_open_file : zeroing journal 2023-09-22T13:07:50.595 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.594+0000 7f1409beb200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:50.595 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.594+0000 7f1409beb200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:50.595 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.594+0000 7f1409beb200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:50.605 DEBUG:teuthology.orchestra.run.smithi169:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-22T13:07:50.674 DEBUG:teuthology.orchestra.run.smithi169:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 7 --monmap /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:50.756 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.755+0000 7f2a62226200 -1 auth: error reading file: /var/lib/ceph/osd/ceph-7/keyring: can't open /var/lib/ceph/osd/ceph-7/keyring: (2) No such file or directory 2023-09-22T13:07:50.757 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.756+0000 7f2a62226200 -1 created new key in keyring /var/lib/ceph/osd/ceph-7/keyring 2023-09-22T13:07:50.763 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.762+0000 7f2a62226200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:50.764 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.762+0000 7f2a62226200 -1 journal FileJournal::_open_file : zeroing journal 2023-09-22T13:07:50.891 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.890+0000 7f2a62226200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:07:50.891 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.890+0000 7f2a62226200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:50.891 INFO:teuthology.orchestra.run.smithi169.stderr:2023-09-22T13:07:50.890+0000 7f2a62226200 -1 journal do_read_entry(4096): bad header magic 2023-09-22T13:07:50.900 DEBUG:teuthology.orchestra.run.smithi169:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-09-22T13:07:50.968 INFO:tasks.ceph:Reading keys from all nodes... 2023-09-22T13:07:50.969 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:50.969 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/mgr/ceph-y/keyring of=/dev/stdout 2023-09-22T13:07:51.002 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:51.002 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-0/keyring of=/dev/stdout 2023-09-22T13:07:51.075 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:51.075 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-1/keyring of=/dev/stdout 2023-09-22T13:07:51.147 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:51.147 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-2/keyring of=/dev/stdout 2023-09-22T13:07:51.218 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:51.219 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-3/keyring of=/dev/stdout 2023-09-22T13:07:51.289 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:51.289 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/mgr/ceph-x/keyring of=/dev/stdout 2023-09-22T13:07:51.317 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:51.359 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/ceph-4/keyring of=/dev/stdout 2023-09-22T13:07:51.387 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:51.387 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/ceph-5/keyring of=/dev/stdout 2023-09-22T13:07:51.455 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:51.455 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/ceph-6/keyring of=/dev/stdout 2023-09-22T13:07:51.521 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:51.522 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/ceph-7/keyring of=/dev/stdout 2023-09-22T13:07:51.589 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:51.589 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/etc/ceph/ceph.client.0.keyring of=/dev/stdout 2023-09-22T13:07:51.610 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:51.610 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/etc/ceph/ceph.client.1.keyring of=/dev/stdout 2023-09-22T13:07:51.647 INFO:tasks.ceph:Adding keys to all mons... 2023-09-22T13:07:51.647 DEBUG:teuthology.orchestra.run.smithi098:> sudo tee -a /etc/ceph/ceph.keyring 2023-09-22T13:07:51.653 DEBUG:teuthology.orchestra.run.smithi169:> sudo tee -a /etc/ceph/ceph.keyring 2023-09-22T13:07:51.690 INFO:teuthology.orchestra.run.smithi098.stdout:[mgr.y] 2023-09-22T13:07:51.691 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQCekQ1lRQx1AxAAoQytgYyiOKXxCoZT5NekbA== 2023-09-22T13:07:51.691 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.0] 2023-09-22T13:07:51.691 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQChkQ1lee8VGRAAILq9NHmWT7cJh4kemr5qBw== 2023-09-22T13:07:51.691 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.1] 2023-09-22T13:07:51.691 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQChkQ1lDLWDLRAAsUwB+yhBEFbZnwn06q4wPQ== 2023-09-22T13:07:51.691 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.2] 2023-09-22T13:07:51.691 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQCikQ1lh2vTAxAAqN6Xr69FgieXyQD5PYvKXA== 2023-09-22T13:07:51.692 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.3] 2023-09-22T13:07:51.692 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQCikQ1ljxJBFhAAN8BIb6XSTRlsFDTexrYEQw== 2023-09-22T13:07:51.692 INFO:teuthology.orchestra.run.smithi098.stdout:[mgr.x] 2023-09-22T13:07:51.692 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQCekQ1lTVtMCxAAKinxZCJflFmtEIEDtyrh0Q== 2023-09-22T13:07:51.692 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.4] 2023-09-22T13:07:51.692 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQClkQ1lcVh/LhAAA1sswLrHITeBMOdd5kMmFg== 2023-09-22T13:07:51.692 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.5] 2023-09-22T13:07:51.692 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQCmkQ1l08KuBxAAJagXhQx18oJYMKhe8oV+jg== 2023-09-22T13:07:51.693 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.6] 2023-09-22T13:07:51.693 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQCmkQ1lwX6oGhAAZdd4OHwL+WmiszfYZYPYjQ== 2023-09-22T13:07:51.693 INFO:teuthology.orchestra.run.smithi098.stdout:[osd.7] 2023-09-22T13:07:51.693 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQCmkQ1ltNUkLRAA4yxt6EP9pFaLyA57+dxv0g== 2023-09-22T13:07:51.693 INFO:teuthology.orchestra.run.smithi098.stdout:[client.0] 2023-09-22T13:07:51.693 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQCekQ1lSeHsDRAAw5sJBFuNI8v3xNLI3F7SyQ== 2023-09-22T13:07:51.693 INFO:teuthology.orchestra.run.smithi098.stdout:[client.1] 2023-09-22T13:07:51.693 INFO:teuthology.orchestra.run.smithi098.stdout: key = AQCekQ1lYx4QERAAjbrwcZfEUPZRU555upYy/g== 2023-09-22T13:07:51.712 INFO:teuthology.orchestra.run.smithi169.stdout:[mgr.y] 2023-09-22T13:07:51.712 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQCekQ1lRQx1AxAAoQytgYyiOKXxCoZT5NekbA== 2023-09-22T13:07:51.713 INFO:teuthology.orchestra.run.smithi169.stdout:[osd.0] 2023-09-22T13:07:51.713 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQChkQ1lee8VGRAAILq9NHmWT7cJh4kemr5qBw== 2023-09-22T13:07:51.713 INFO:teuthology.orchestra.run.smithi169.stdout:[osd.1] 2023-09-22T13:07:51.713 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQChkQ1lDLWDLRAAsUwB+yhBEFbZnwn06q4wPQ== 2023-09-22T13:07:51.713 INFO:teuthology.orchestra.run.smithi169.stdout:[osd.2] 2023-09-22T13:07:51.713 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQCikQ1lh2vTAxAAqN6Xr69FgieXyQD5PYvKXA== 2023-09-22T13:07:51.713 INFO:teuthology.orchestra.run.smithi169.stdout:[osd.3] 2023-09-22T13:07:51.714 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQCikQ1ljxJBFhAAN8BIb6XSTRlsFDTexrYEQw== 2023-09-22T13:07:51.714 INFO:teuthology.orchestra.run.smithi169.stdout:[mgr.x] 2023-09-22T13:07:51.714 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQCekQ1lTVtMCxAAKinxZCJflFmtEIEDtyrh0Q== 2023-09-22T13:07:51.714 INFO:teuthology.orchestra.run.smithi169.stdout:[osd.4] 2023-09-22T13:07:51.714 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQClkQ1lcVh/LhAAA1sswLrHITeBMOdd5kMmFg== 2023-09-22T13:07:51.714 INFO:teuthology.orchestra.run.smithi169.stdout:[osd.5] 2023-09-22T13:07:51.714 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQCmkQ1l08KuBxAAJagXhQx18oJYMKhe8oV+jg== 2023-09-22T13:07:51.714 INFO:teuthology.orchestra.run.smithi169.stdout:[osd.6] 2023-09-22T13:07:51.714 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQCmkQ1lwX6oGhAAZdd4OHwL+WmiszfYZYPYjQ== 2023-09-22T13:07:51.715 INFO:teuthology.orchestra.run.smithi169.stdout:[osd.7] 2023-09-22T13:07:51.715 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQCmkQ1ltNUkLRAA4yxt6EP9pFaLyA57+dxv0g== 2023-09-22T13:07:51.715 INFO:teuthology.orchestra.run.smithi169.stdout:[client.0] 2023-09-22T13:07:51.715 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQCekQ1lSeHsDRAAw5sJBFuNI8v3xNLI3F7SyQ== 2023-09-22T13:07:51.715 INFO:teuthology.orchestra.run.smithi169.stdout:[client.1] 2023-09-22T13:07:51.715 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQCekQ1lYx4QERAAjbrwcZfEUPZRU555upYy/g== 2023-09-22T13:07:51.716 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:07:51.737 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:07:51.806 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:51.809 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:51.857 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:51.860 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:51.908 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:51.911 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:51.959 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:51.962 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:52.010 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:07:52.012 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2023-09-22T13:07:52.060 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:52.063 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.4 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:52.111 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:52.114 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.5 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:52.162 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:52.164 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.6 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:52.213 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:52.215 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.7 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2023-09-22T13:07:52.264 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:07:52.266 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:07:52.317 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:07:52.319 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2023-09-22T13:07:52.370 INFO:tasks.ceph:Running mkfs on mon nodes... 2023-09-22T13:07:52.370 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/mon/ceph-a 2023-09-22T13:07:52.397 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i a --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2023-09-22T13:07:52.491 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-a 2023-09-22T13:07:52.517 DEBUG:teuthology.orchestra.run.smithi098:> sudo mkdir -p /var/lib/ceph/mon/ceph-c 2023-09-22T13:07:52.584 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i c --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2023-09-22T13:07:52.678 DEBUG:teuthology.orchestra.run.smithi098:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-c 2023-09-22T13:07:52.704 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /var/lib/ceph/mon/ceph-b 2023-09-22T13:07:52.733 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i b --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2023-09-22T13:07:52.828 DEBUG:teuthology.orchestra.run.smithi169:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-b 2023-09-22T13:07:52.855 DEBUG:teuthology.orchestra.run.smithi098:> rm -- /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:52.857 DEBUG:teuthology.orchestra.run.smithi169:> rm -- /home/ubuntu/cephtest/ceph.monmap 2023-09-22T13:07:52.913 INFO:tasks.ceph:Starting mon daemons in cluster ceph... 2023-09-22T13:07:52.914 INFO:tasks.ceph.mon.a:Restarting daemon 2023-09-22T13:07:52.914 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2023-09-22T13:07:52.916 INFO:tasks.ceph.mon.a:Started 2023-09-22T13:07:52.916 INFO:tasks.ceph.mon.c:Restarting daemon 2023-09-22T13:07:52.917 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2023-09-22T13:07:52.918 INFO:tasks.ceph.mon.c:Started 2023-09-22T13:07:52.919 INFO:tasks.ceph.mon.b:Restarting daemon 2023-09-22T13:07:52.919 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2023-09-22T13:07:52.956 INFO:tasks.ceph.mon.b:Started 2023-09-22T13:07:52.956 INFO:tasks.ceph:Starting mgr daemons in cluster ceph... 2023-09-22T13:07:52.956 INFO:tasks.ceph.mgr.y:Restarting daemon 2023-09-22T13:07:52.956 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2023-09-22T13:07:52.959 INFO:tasks.ceph.mgr.y:Started 2023-09-22T13:07:52.959 INFO:tasks.ceph.mgr.x:Restarting daemon 2023-09-22T13:07:52.959 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2023-09-22T13:07:52.961 INFO:tasks.ceph.mgr.x:Started 2023-09-22T13:07:52.961 INFO:tasks.ceph:Setting crush tunables to default 2023-09-22T13:07:52.961 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph osd crush tunables default 2023-09-22T13:07:58.375 INFO:teuthology.orchestra.run.smithi098.stderr:adjusted tunables profile to default 2023-09-22T13:07:58.386 INFO:tasks.ceph:Starting osd daemons in cluster ceph... 2023-09-22T13:07:58.386 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:58.386 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-0/fsid of=/dev/stdout 2023-09-22T13:07:58.415 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:58.415 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-1/fsid of=/dev/stdout 2023-09-22T13:07:58.420 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:07:58.419+0000 7fd7cd96c680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-22T13:07:58.442 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:07:58.442+0000 7fa70c1be680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-22T13:07:58.468 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:58.469 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-2/fsid of=/dev/stdout 2023-09-22T13:07:58.490 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:07:58.489+0000 7fd7cd96c680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-22T13:07:58.514 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:07:58.515+0000 7fa70c1be680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-22T13:07:58.536 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:07:58.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd if=/var/lib/ceph/osd/ceph-3/fsid of=/dev/stdout 2023-09-22T13:07:58.603 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:58.604 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/ceph-4/fsid of=/dev/stdout 2023-09-22T13:07:58.631 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:58.631 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/ceph-5/fsid of=/dev/stdout 2023-09-22T13:07:58.699 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:58.700 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/ceph-6/fsid of=/dev/stdout 2023-09-22T13:07:58.767 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-09-22T13:07:58.767 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/var/lib/ceph/osd/ceph-7/fsid of=/dev/stdout 2023-09-22T13:07:58.834 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster ceph osd new 5482e19a-2a28-4455-8110-fb834d873a55 0 2023-09-22T13:07:58.930 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:07:58.930+0000 7fa70c1be680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-22T13:07:58.935 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:07:58.934+0000 7fd7cd96c680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-22T13:07:59.084 INFO:teuthology.orchestra.run.smithi169.stdout:0 2023-09-22T13:07:59.096 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster ceph osd new d9714f8e-eda9-40fe-a1bd-eaff19e52ba6 1 2023-09-22T13:07:59.306 INFO:teuthology.orchestra.run.smithi169.stdout:1 2023-09-22T13:07:59.319 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster ceph osd new f224f2a4-77ec-49e2-8a73-2a191cd83f27 2 2023-09-22T13:07:59.545 INFO:teuthology.orchestra.run.smithi169.stdout:2 2023-09-22T13:07:59.558 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster ceph osd new a43653eb-4a91-4b86-a125-e274d80ed338 3 2023-09-22T13:07:59.660 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:07:59.660+0000 7fa70c1be680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-22T13:07:59.720 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:07:59.719+0000 7fd7cd96c680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-22T13:07:59.728 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:07:59.729+0000 7fa70c1be680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-22T13:07:59.788 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:07:59.787+0000 7fd7cd96c680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-22T13:07:59.790 INFO:teuthology.orchestra.run.smithi169.stdout:3 2023-09-22T13:07:59.802 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster ceph osd new 783c24fd-c283-436f-afd6-77320755247e 4 2023-09-22T13:07:59.914 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:07:59.914+0000 7fa70c1be680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-22T13:07:59.991 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:07:59.991+0000 7fd7cd96c680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-22T13:08:00.012 INFO:teuthology.orchestra.run.smithi169.stdout:4 2023-09-22T13:08:00.025 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster ceph osd new e9410a8c-5c45-404a-b5e2-1686fe079232 5 2023-09-22T13:08:00.240 INFO:teuthology.orchestra.run.smithi169.stdout:5 2023-09-22T13:08:00.253 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster ceph osd new 0fe4f080-1273-4d09-a804-4fc9efb89815 6 2023-09-22T13:08:00.352 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:00.352+0000 7fa70c1be680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-22T13:08:00.454 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:00.454+0000 7fa70c1be680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:08:00.457 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:00.457+0000 7fd7cd96c680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-22T13:08:00.466 INFO:teuthology.orchestra.run.smithi169.stdout:6 2023-09-22T13:08:00.479 DEBUG:teuthology.orchestra.run.smithi169:> sudo ceph --cluster ceph osd new f0dedb37-14f4-4e9e-ab7d-1cf0de58597b 7 2023-09-22T13:08:00.566 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:00.566+0000 7fd7cd96c680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:08:00.605 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:00.605+0000 7fa70c1be680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-22T13:08:00.682 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:00.682+0000 7fa70c1be680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-22T13:08:00.687 INFO:teuthology.orchestra.run.smithi169.stdout:7 2023-09-22T13:08:00.700 INFO:tasks.ceph.osd.0:Restarting daemon 2023-09-22T13:08:00.700 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2023-09-22T13:08:00.703 INFO:tasks.ceph.osd.0:Started 2023-09-22T13:08:00.703 INFO:tasks.ceph.osd.1:Restarting daemon 2023-09-22T13:08:00.704 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2023-09-22T13:08:00.706 INFO:tasks.ceph.osd.1:Started 2023-09-22T13:08:00.706 INFO:tasks.ceph.osd.2:Restarting daemon 2023-09-22T13:08:00.706 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2023-09-22T13:08:00.709 INFO:tasks.ceph.osd.2:Started 2023-09-22T13:08:00.709 INFO:tasks.ceph.osd.3:Restarting daemon 2023-09-22T13:08:00.709 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2023-09-22T13:08:00.712 INFO:tasks.ceph.osd.3:Started 2023-09-22T13:08:00.713 INFO:tasks.ceph.osd.4:Restarting daemon 2023-09-22T13:08:00.713 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 4 2023-09-22T13:08:00.715 INFO:tasks.ceph.osd.4:Started 2023-09-22T13:08:00.716 INFO:tasks.ceph.osd.5:Restarting daemon 2023-09-22T13:08:00.716 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 5 2023-09-22T13:08:00.718 INFO:tasks.ceph.osd.5:Started 2023-09-22T13:08:00.718 INFO:tasks.ceph.osd.6:Restarting daemon 2023-09-22T13:08:00.719 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 6 2023-09-22T13:08:00.721 INFO:tasks.ceph.osd.6:Started 2023-09-22T13:08:00.721 INFO:tasks.ceph.osd.7:Restarting daemon 2023-09-22T13:08:00.722 DEBUG:teuthology.orchestra.run.smithi169:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 7 2023-09-22T13:08:00.724 INFO:tasks.ceph.osd.7:Started 2023-09-22T13:08:00.725 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-09-22T13:08:00.727 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:00.727+0000 7fd7cd96c680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-22T13:08:00.779 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:00.779+0000 7fa70c1be680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-22T13:08:00.792 INFO:tasks.ceph.osd.1.smithi098.stderr:2023-09-22T13:08:00.792+0000 7fa4f344c200 -1 Falling back to public interface 2023-09-22T13:08:00.797 INFO:tasks.ceph.osd.4.smithi169.stderr:2023-09-22T13:08:00.797+0000 7fc7cc2d4200 -1 Falling back to public interface 2023-09-22T13:08:00.804 INFO:tasks.ceph.osd.6.smithi169.stderr:2023-09-22T13:08:00.804+0000 7f772c34a200 -1 Falling back to public interface 2023-09-22T13:08:00.808 INFO:tasks.ceph.osd.4.smithi169.stderr:2023-09-22T13:08:00.808+0000 7fc7cc2d4200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:08:00.809 INFO:tasks.ceph.osd.4.smithi169.stderr:2023-09-22T13:08:00.808+0000 7fc7cc2d4200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.809 INFO:tasks.ceph.osd.4.smithi169.stderr:2023-09-22T13:08:00.808+0000 7fc7cc2d4200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.809 INFO:tasks.ceph.osd.1.smithi098.stderr:2023-09-22T13:08:00.809+0000 7fa4f344c200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:08:00.810 INFO:tasks.ceph.osd.1.smithi098.stderr:2023-09-22T13:08:00.809+0000 7fa4f344c200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.810 INFO:tasks.ceph.osd.1.smithi098.stderr:2023-09-22T13:08:00.809+0000 7fa4f344c200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.811 INFO:tasks.ceph.osd.0.smithi098.stderr:2023-09-22T13:08:00.810+0000 7f957e5ad200 -1 Falling back to public interface 2023-09-22T13:08:00.816 INFO:tasks.ceph.osd.4.smithi169.stderr:2023-09-22T13:08:00.816+0000 7fc7cc2d4200 -1 osd.4 0 log_to_monitors {default=true} 2023-09-22T13:08:00.817 INFO:tasks.ceph.osd.1.smithi098.stderr:2023-09-22T13:08:00.817+0000 7fa4f344c200 -1 osd.1 0 log_to_monitors {default=true} 2023-09-22T13:08:00.821 INFO:tasks.ceph.osd.2.smithi098.stderr:2023-09-22T13:08:00.821+0000 7fbd589c5200 -1 Falling back to public interface 2023-09-22T13:08:00.823 INFO:tasks.ceph.osd.3.smithi098.stderr:2023-09-22T13:08:00.823+0000 7f57f9089200 -1 Falling back to public interface 2023-09-22T13:08:00.824 INFO:tasks.ceph.osd.5.smithi169.stderr:2023-09-22T13:08:00.824+0000 7f2ac3317200 -1 Falling back to public interface 2023-09-22T13:08:00.825 INFO:tasks.ceph.osd.6.smithi169.stderr:2023-09-22T13:08:00.825+0000 7f772c34a200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:08:00.825 INFO:tasks.ceph.osd.6.smithi169.stderr:2023-09-22T13:08:00.825+0000 7f772c34a200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.826 INFO:tasks.ceph.osd.6.smithi169.stderr:2023-09-22T13:08:00.825+0000 7f772c34a200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.827 INFO:tasks.ceph.osd.7.smithi169.stderr:2023-09-22T13:08:00.827+0000 7f745edfa200 -1 Falling back to public interface 2023-09-22T13:08:00.828 INFO:tasks.ceph.osd.0.smithi098.stderr:2023-09-22T13:08:00.827+0000 7f957e5ad200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:08:00.833 INFO:tasks.ceph.osd.0.smithi098.stderr:2023-09-22T13:08:00.829+0000 7f957e5ad200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.833 INFO:tasks.ceph.osd.0.smithi098.stderr:2023-09-22T13:08:00.829+0000 7f957e5ad200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.833 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:00.833+0000 7fd7cd96c680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-22T13:08:00.834 INFO:tasks.ceph.osd.6.smithi169.stderr:2023-09-22T13:08:00.833+0000 7f772c34a200 -1 osd.6 0 log_to_monitors {default=true} 2023-09-22T13:08:00.837 INFO:tasks.ceph.osd.0.smithi098.stderr:2023-09-22T13:08:00.837+0000 7f957e5ad200 -1 osd.0 0 log_to_monitors {default=true} 2023-09-22T13:08:00.842 INFO:tasks.ceph.osd.7.smithi169.stderr:2023-09-22T13:08:00.842+0000 7f745edfa200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:08:00.842 INFO:tasks.ceph.osd.7.smithi169.stderr:2023-09-22T13:08:00.842+0000 7f745edfa200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.842 INFO:tasks.ceph.osd.7.smithi169.stderr:2023-09-22T13:08:00.842+0000 7f745edfa200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.843 INFO:tasks.ceph.osd.5.smithi169.stderr:2023-09-22T13:08:00.843+0000 7f2ac3317200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:08:00.843 INFO:tasks.ceph.osd.5.smithi169.stderr:2023-09-22T13:08:00.843+0000 7f2ac3317200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.843 INFO:tasks.ceph.osd.5.smithi169.stderr:2023-09-22T13:08:00.843+0000 7f2ac3317200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.844 INFO:tasks.ceph.osd.3.smithi098.stderr:2023-09-22T13:08:00.844+0000 7f57f9089200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:08:00.845 INFO:tasks.ceph.osd.3.smithi098.stderr:2023-09-22T13:08:00.844+0000 7f57f9089200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.845 INFO:tasks.ceph.osd.3.smithi098.stderr:2023-09-22T13:08:00.844+0000 7f57f9089200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.847 INFO:tasks.ceph.osd.2.smithi098.stderr:2023-09-22T13:08:00.847+0000 7fbd589c5200 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2023-09-22T13:08:00.847 INFO:tasks.ceph.osd.2.smithi098.stderr:2023-09-22T13:08:00.847+0000 7fbd589c5200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.847 INFO:tasks.ceph.osd.2.smithi098.stderr:2023-09-22T13:08:00.847+0000 7fbd589c5200 -1 journal do_read_entry(8192): bad header magic 2023-09-22T13:08:00.848 INFO:tasks.ceph.osd.7.smithi169.stderr:2023-09-22T13:08:00.848+0000 7f745edfa200 -1 osd.7 0 log_to_monitors {default=true} 2023-09-22T13:08:00.851 INFO:tasks.ceph.osd.5.smithi169.stderr:2023-09-22T13:08:00.851+0000 7f2ac3317200 -1 osd.5 0 log_to_monitors {default=true} 2023-09-22T13:08:00.853 INFO:tasks.ceph.osd.3.smithi098.stderr:2023-09-22T13:08:00.853+0000 7f57f9089200 -1 osd.3 0 log_to_monitors {default=true} 2023-09-22T13:08:00.855 INFO:tasks.ceph.osd.2.smithi098.stderr:2023-09-22T13:08:00.855+0000 7fbd589c5200 -1 osd.2 0 log_to_monitors {default=true} 2023-09-22T13:08:00.914 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:00.914+0000 7fd7cd96c680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-22T13:08:00.951 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:08:00.951 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":10,"fsid":"583b11f8-631b-40f6-ac70-b90b0e6977fb","created":"2023-09-22T13:07:58.230141+0000","modified":"2023-09-22T13:08:00.680979+0000","last_up_change":"0.000000","last_in_change":"2023-09-22T13:08:00.680979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[],"osds":[{"osd":0,"uuid":"5482e19a-2a28-4455-8110-fb834d873a55","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"d9714f8e-eda9-40fe-a1bd-eaff19e52ba6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f224f2a4-77ec-49e2-8a73-2a191cd83f27","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"a43653eb-4a91-4b86-a125-e274d80ed338","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"783c24fd-c283-436f-afd6-77320755247e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"e9410a8c-5c45-404a-b5e2-1686fe079232","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"0fe4f080-1273-4d09-a804-4fc9efb89815","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"f0dedb37-14f4-4e9e-ab7d-1cf0de58597b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:08:00.963 INFO:tasks.ceph.ceph_manager.ceph:[] 2023-09-22T13:08:00.963 INFO:tasks.ceph:Waiting for OSDs to come up 2023-09-22T13:08:01.079 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:01.079+0000 7fa70c1be680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-22T13:08:01.174 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:01.174+0000 7fd7cd96c680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-22T13:08:01.279 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:01.280+0000 7fa70c1be680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-22T13:08:01.376 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:01.376+0000 7fd7cd96c680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-22T13:08:01.769 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-22T13:08:01.945 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:01.945+0000 7fa70c1be680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-22T13:08:01.961 INFO:teuthology.misc.health.smithi098.stdout: 2023-09-22T13:08:01.961 INFO:teuthology.misc.health.smithi098.stdout:{"epoch":11,"fsid":"583b11f8-631b-40f6-ac70-b90b0e6977fb","created":"2023-09-22T13:07:58.230141+0000","modified":"2023-09-22T13:08:01.681875+0000","last_up_change":"0.000000","last_in_change":"2023-09-22T13:08:00.680979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[],"osds":[{"osd":0,"uuid":"5482e19a-2a28-4455-8110-fb834d873a55","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"d9714f8e-eda9-40fe-a1bd-eaff19e52ba6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f224f2a4-77ec-49e2-8a73-2a191cd83f27","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"a43653eb-4a91-4b86-a125-e274d80ed338","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":4,"uuid":"783c24fd-c283-436f-afd6-77320755247e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":5,"uuid":"e9410a8c-5c45-404a-b5e2-1686fe079232","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":6,"uuid":"0fe4f080-1273-4d09-a804-4fc9efb89815","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":7,"uuid":"f0dedb37-14f4-4e9e-ab7d-1cf0de58597b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:08:01.972 DEBUG:teuthology.misc:0 of 8 OSDs are up 2023-09-22T13:08:02.016 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:02.017+0000 7fa70c1be680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-22T13:08:02.040 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:02.040+0000 7fd7cd96c680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-22T13:08:02.094 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:02.094+0000 7fa70c1be680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-22T13:08:02.109 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:02.109+0000 7fd7cd96c680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-22T13:08:02.188 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:02.188+0000 7fd7cd96c680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-22T13:08:02.241 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:02.241+0000 7fa70c1be680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-22T13:08:02.312 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:02.312+0000 7fa70c1be680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-22T13:08:02.339 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:02.339+0000 7fd7cd96c680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-22T13:08:02.410 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:02.410+0000 7fd7cd96c680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-22T13:08:02.425 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:02.425+0000 7fa70c1be680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-22T13:08:02.527 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:02.527+0000 7fd7cd96c680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-22T13:08:02.532 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:02.532+0000 7fa70c1be680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:08:02.636 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:02.636+0000 7fd7cd96c680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-22T13:08:02.688 INFO:tasks.ceph.osd.2.smithi098.stderr:2023-09-22T13:08:02.688+0000 7fbd4fb67700 -1 osd.2 0 waiting for initial osdmap 2023-09-22T13:08:02.688 INFO:tasks.ceph.osd.7.smithi169.stderr:2023-09-22T13:08:02.688+0000 7f7455f9c700 -1 osd.7 0 waiting for initial osdmap 2023-09-22T13:08:02.689 INFO:tasks.ceph.osd.3.smithi098.stderr:2023-09-22T13:08:02.688+0000 7f57f022b700 -1 osd.3 0 waiting for initial osdmap 2023-09-22T13:08:02.689 INFO:tasks.ceph.osd.5.smithi169.stderr:2023-09-22T13:08:02.689+0000 7f2aba4b9700 -1 osd.5 0 waiting for initial osdmap 2023-09-22T13:08:02.690 INFO:tasks.ceph.osd.7.smithi169.stderr:2023-09-22T13:08:02.690+0000 7f7451f17700 -1 osd.7 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:08:02.690 INFO:tasks.ceph.osd.2.smithi098.stderr:2023-09-22T13:08:02.690+0000 7fbd4bae2700 -1 osd.2 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:08:02.691 INFO:tasks.ceph.osd.5.smithi169.stderr:2023-09-22T13:08:02.691+0000 7f2ab6434700 -1 osd.5 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:08:02.697 INFO:tasks.ceph.osd.0.smithi098.stderr:2023-09-22T13:08:02.696+0000 7f957574f700 -1 osd.0 0 waiting for initial osdmap 2023-09-22T13:08:02.697 INFO:tasks.ceph.osd.6.smithi169.stderr:2023-09-22T13:08:02.696+0000 7f77234ec700 -1 osd.6 0 waiting for initial osdmap 2023-09-22T13:08:02.697 INFO:tasks.ceph.osd.1.smithi098.stderr:2023-09-22T13:08:02.696+0000 7fa4ea5ee700 -1 osd.1 0 waiting for initial osdmap 2023-09-22T13:08:02.698 INFO:tasks.ceph.osd.4.smithi169.stderr:2023-09-22T13:08:02.696+0000 7fc7c3476700 -1 osd.4 0 waiting for initial osdmap 2023-09-22T13:08:02.699 INFO:tasks.ceph.osd.0.smithi098.stderr:2023-09-22T13:08:02.698+0000 7f95716ca700 -1 osd.0 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:08:02.699 INFO:tasks.ceph.osd.4.smithi169.stderr:2023-09-22T13:08:02.698+0000 7fc7bf3f1700 -1 osd.4 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:08:02.699 INFO:tasks.ceph.osd.1.smithi098.stderr:2023-09-22T13:08:02.698+0000 7fa4e6569700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:08:02.700 INFO:tasks.ceph.osd.6.smithi169.stderr:2023-09-22T13:08:02.698+0000 7f771f467700 -1 osd.6 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:08:02.700 INFO:tasks.ceph.osd.3.smithi098.stderr:2023-09-22T13:08:02.699+0000 7f57ec1a6700 -1 osd.3 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-22T13:08:02.853 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:02.853+0000 7fa70c1be680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-22T13:08:02.922 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:02.922+0000 7fa70c1be680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-22T13:08:02.967 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:02.967+0000 7fd7cd96c680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-22T13:08:03.038 INFO:tasks.ceph.mgr.x.smithi169.stderr:2023-09-22T13:08:03.038+0000 7fd7cd96c680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-22T13:08:03.939 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:03.939+0000 7fa6c16b9700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-22T13:08:03.939 INFO:tasks.ceph.mgr.y.smithi098.stderr:2023-09-22T13:08:03.939+0000 7fa6c16b9700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-22T13:08:08.779 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-22T13:08:09.075 INFO:teuthology.misc.health.smithi098.stdout: 2023-09-22T13:08:09.075 INFO:teuthology.misc.health.smithi098.stdout:{"epoch":16,"fsid":"583b11f8-631b-40f6-ac70-b90b0e6977fb","created":"2023-09-22T13:07:58.230141+0000","modified":"2023-09-22T13:08:06.712391+0000","last_up_change":"2023-09-22T13:08:03.695960+0000","last_in_change":"2023-09-22T13:08:00.680979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-22T13:08:03.733606+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"5482e19a-2a28-4455-8110-fb834d873a55","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":140877},{"type":"v1","addr":"172.21.15.98:6809","nonce":140877}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":140877},{"type":"v1","addr":"172.21.15.98:6811","nonce":140877}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":140877},{"type":"v1","addr":"172.21.15.98:6815","nonce":140877}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":140877},{"type":"v1","addr":"172.21.15.98:6813","nonce":140877}]},"public_addr":"172.21.15.98:6809/140877","cluster_addr":"172.21.15.98:6811/140877","heartbeat_back_addr":"172.21.15.98:6815/140877","heartbeat_front_addr":"172.21.15.98:6813/140877","state":["exists","up"]},{"osd":1,"uuid":"d9714f8e-eda9-40fe-a1bd-eaff19e52ba6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6800","nonce":140876},{"type":"v1","addr":"172.21.15.98:6801","nonce":140876}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":140876},{"type":"v1","addr":"172.21.15.98:6803","nonce":140876}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":140876},{"type":"v1","addr":"172.21.15.98:6807","nonce":140876}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":140876},{"type":"v1","addr":"172.21.15.98:6805","nonce":140876}]},"public_addr":"172.21.15.98:6801/140876","cluster_addr":"172.21.15.98:6803/140876","heartbeat_back_addr":"172.21.15.98:6807/140876","heartbeat_front_addr":"172.21.15.98:6805/140876","state":["exists","up"]},{"osd":2,"uuid":"f224f2a4-77ec-49e2-8a73-2a191cd83f27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":140892},{"type":"v1","addr":"172.21.15.98:6817","nonce":140892}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":140892},{"type":"v1","addr":"172.21.15.98:6819","nonce":140892}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":140892},{"type":"v1","addr":"172.21.15.98:6831","nonce":140892}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6821","nonce":140892},{"type":"v1","addr":"172.21.15.98:6829","nonce":140892}]},"public_addr":"172.21.15.98:6817/140892","cluster_addr":"172.21.15.98:6819/140892","heartbeat_back_addr":"172.21.15.98:6831/140892","heartbeat_front_addr":"172.21.15.98:6829/140892","state":["exists","up"]},{"osd":3,"uuid":"a43653eb-4a91-4b86-a125-e274d80ed338","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":140891},{"type":"v1","addr":"172.21.15.98:6822","nonce":140891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6823","nonce":140891},{"type":"v1","addr":"172.21.15.98:6824","nonce":140891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6827","nonce":140891},{"type":"v1","addr":"172.21.15.98:6828","nonce":140891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6825","nonce":140891},{"type":"v1","addr":"172.21.15.98:6826","nonce":140891}]},"public_addr":"172.21.15.98:6822/140891","cluster_addr":"172.21.15.98:6824/140891","heartbeat_back_addr":"172.21.15.98:6828/140891","heartbeat_front_addr":"172.21.15.98:6826/140891","state":["exists","up"]},{"osd":4,"uuid":"783c24fd-c283-436f-afd6-77320755247e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":140693},{"type":"v1","addr":"172.21.15.169:6801","nonce":140693}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":140693},{"type":"v1","addr":"172.21.15.169:6803","nonce":140693}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":140693},{"type":"v1","addr":"172.21.15.169:6807","nonce":140693}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":140693},{"type":"v1","addr":"172.21.15.169:6805","nonce":140693}]},"public_addr":"172.21.15.169:6801/140693","cluster_addr":"172.21.15.169:6803/140693","heartbeat_back_addr":"172.21.15.169:6807/140693","heartbeat_front_addr":"172.21.15.169:6805/140693","state":["exists","up"]},{"osd":5,"uuid":"e9410a8c-5c45-404a-b5e2-1686fe079232","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":140717},{"type":"v1","addr":"172.21.15.169:6817","nonce":140717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":140717},{"type":"v1","addr":"172.21.15.169:6819","nonce":140717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":140717},{"type":"v1","addr":"172.21.15.169:6827","nonce":140717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":140717},{"type":"v1","addr":"172.21.15.169:6822","nonce":140717}]},"public_addr":"172.21.15.169:6817/140717","cluster_addr":"172.21.15.169:6819/140717","heartbeat_back_addr":"172.21.15.169:6827/140717","heartbeat_front_addr":"172.21.15.169:6822/140717","state":["exists","up"]},{"osd":6,"uuid":"0fe4f080-1273-4d09-a804-4fc9efb89815","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":140694},{"type":"v1","addr":"172.21.15.169:6809","nonce":140694}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":140694},{"type":"v1","addr":"172.21.15.169:6811","nonce":140694}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":140694},{"type":"v1","addr":"172.21.15.169:6815","nonce":140694}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":140694},{"type":"v1","addr":"172.21.15.169:6813","nonce":140694}]},"public_addr":"172.21.15.169:6809/140694","cluster_addr":"172.21.15.169:6811/140694","heartbeat_back_addr":"172.21.15.169:6815/140694","heartbeat_front_addr":"172.21.15.169:6813/140694","state":["exists","up"]},{"osd":7,"uuid":"f0dedb37-14f4-4e9e-ab7d-1cf0de58597b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6821","nonce":140771},{"type":"v1","addr":"172.21.15.169:6823","nonce":140771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6825","nonce":140771},{"type":"v1","addr":"172.21.15.169:6826","nonce":140771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":140771},{"type":"v1","addr":"172.21.15.169:6831","nonce":140771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":140771},{"type":"v1","addr":"172.21.15.169:6829","nonce":140771}]},"public_addr":"172.21.15.169:6823/140771","cluster_addr":"172.21.15.169:6826/140771","heartbeat_back_addr":"172.21.15.169:6831/140771","heartbeat_front_addr":"172.21.15.169:6829/140771","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.821296+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.796648+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.816487+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.889969+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.819961+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.838840+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.817619+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.849885+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:08:09.087 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-22T13:08:09.087 INFO:tasks.ceph:Creating RBD pool 2023-09-22T13:08:09.087 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph osd pool create rbd 8 2023-09-22T13:08:10.026 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'rbd' created 2023-09-22T13:08:10.039 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-22T13:08:11.036 INFO:teuthology.orchestra.run.smithi098.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-22T13:08:11.049 INFO:tasks.ceph:Starting mds daemons in cluster ceph... 2023-09-22T13:08:11.050 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-22T13:08:11.050 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2023-09-22T13:08:11.050 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2023-09-22T13:08:11.050 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2023-09-22T13:08:11.392 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:08:11.403 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":4,"active_gid":4107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6832","nonce":140579},{"type":"v1","addr":"172.21.15.98:6833","nonce":140579}]},"active_addr":"172.21.15.98:6833/140579","active_change":"2023-09-22T13:08:02.924201+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":4102,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.98:0","nonce":1259884515}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.98:0","nonce":113390478}]}]} 2023-09-22T13:08:11.409 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2023-09-22T13:08:11.409 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2023-09-22T13:08:11.409 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-09-22T13:08:11.694 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:08:11.695 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":18,"fsid":"583b11f8-631b-40f6-ac70-b90b0e6977fb","created":"2023-09-22T13:07:58.230141+0000","modified":"2023-09-22T13:08:11.021187+0000","last_up_change":"2023-09-22T13:08:03.695960+0000","last_in_change":"2023-09-22T13:08:00.680979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-22T13:08:03.733606+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-22T13:08:09.395420+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"5482e19a-2a28-4455-8110-fb834d873a55","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":140877},{"type":"v1","addr":"172.21.15.98:6809","nonce":140877}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":140877},{"type":"v1","addr":"172.21.15.98:6811","nonce":140877}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":140877},{"type":"v1","addr":"172.21.15.98:6815","nonce":140877}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":140877},{"type":"v1","addr":"172.21.15.98:6813","nonce":140877}]},"public_addr":"172.21.15.98:6809/140877","cluster_addr":"172.21.15.98:6811/140877","heartbeat_back_addr":"172.21.15.98:6815/140877","heartbeat_front_addr":"172.21.15.98:6813/140877","state":["exists","up"]},{"osd":1,"uuid":"d9714f8e-eda9-40fe-a1bd-eaff19e52ba6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6800","nonce":140876},{"type":"v1","addr":"172.21.15.98:6801","nonce":140876}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":140876},{"type":"v1","addr":"172.21.15.98:6803","nonce":140876}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":140876},{"type":"v1","addr":"172.21.15.98:6807","nonce":140876}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":140876},{"type":"v1","addr":"172.21.15.98:6805","nonce":140876}]},"public_addr":"172.21.15.98:6801/140876","cluster_addr":"172.21.15.98:6803/140876","heartbeat_back_addr":"172.21.15.98:6807/140876","heartbeat_front_addr":"172.21.15.98:6805/140876","state":["exists","up"]},{"osd":2,"uuid":"f224f2a4-77ec-49e2-8a73-2a191cd83f27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":140892},{"type":"v1","addr":"172.21.15.98:6817","nonce":140892}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":140892},{"type":"v1","addr":"172.21.15.98:6819","nonce":140892}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":140892},{"type":"v1","addr":"172.21.15.98:6831","nonce":140892}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6821","nonce":140892},{"type":"v1","addr":"172.21.15.98:6829","nonce":140892}]},"public_addr":"172.21.15.98:6817/140892","cluster_addr":"172.21.15.98:6819/140892","heartbeat_back_addr":"172.21.15.98:6831/140892","heartbeat_front_addr":"172.21.15.98:6829/140892","state":["exists","up"]},{"osd":3,"uuid":"a43653eb-4a91-4b86-a125-e274d80ed338","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":17,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":140891},{"type":"v1","addr":"172.21.15.98:6822","nonce":140891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6823","nonce":140891},{"type":"v1","addr":"172.21.15.98:6824","nonce":140891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6827","nonce":140891},{"type":"v1","addr":"172.21.15.98:6828","nonce":140891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6825","nonce":140891},{"type":"v1","addr":"172.21.15.98:6826","nonce":140891}]},"public_addr":"172.21.15.98:6822/140891","cluster_addr":"172.21.15.98:6824/140891","heartbeat_back_addr":"172.21.15.98:6828/140891","heartbeat_front_addr":"172.21.15.98:6826/140891","state":["exists","up"]},{"osd":4,"uuid":"783c24fd-c283-436f-afd6-77320755247e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":17,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":140693},{"type":"v1","addr":"172.21.15.169:6801","nonce":140693}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":140693},{"type":"v1","addr":"172.21.15.169:6803","nonce":140693}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":140693},{"type":"v1","addr":"172.21.15.169:6807","nonce":140693}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":140693},{"type":"v1","addr":"172.21.15.169:6805","nonce":140693}]},"public_addr":"172.21.15.169:6801/140693","cluster_addr":"172.21.15.169:6803/140693","heartbeat_back_addr":"172.21.15.169:6807/140693","heartbeat_front_addr":"172.21.15.169:6805/140693","state":["exists","up"]},{"osd":5,"uuid":"e9410a8c-5c45-404a-b5e2-1686fe079232","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":17,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":140717},{"type":"v1","addr":"172.21.15.169:6817","nonce":140717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":140717},{"type":"v1","addr":"172.21.15.169:6819","nonce":140717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":140717},{"type":"v1","addr":"172.21.15.169:6827","nonce":140717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":140717},{"type":"v1","addr":"172.21.15.169:6822","nonce":140717}]},"public_addr":"172.21.15.169:6817/140717","cluster_addr":"172.21.15.169:6819/140717","heartbeat_back_addr":"172.21.15.169:6827/140717","heartbeat_front_addr":"172.21.15.169:6822/140717","state":["exists","up"]},{"osd":6,"uuid":"0fe4f080-1273-4d09-a804-4fc9efb89815","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":140694},{"type":"v1","addr":"172.21.15.169:6809","nonce":140694}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":140694},{"type":"v1","addr":"172.21.15.169:6811","nonce":140694}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":140694},{"type":"v1","addr":"172.21.15.169:6815","nonce":140694}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":140694},{"type":"v1","addr":"172.21.15.169:6813","nonce":140694}]},"public_addr":"172.21.15.169:6809/140694","cluster_addr":"172.21.15.169:6811/140694","heartbeat_back_addr":"172.21.15.169:6815/140694","heartbeat_front_addr":"172.21.15.169:6813/140694","state":["exists","up"]},{"osd":7,"uuid":"f0dedb37-14f4-4e9e-ab7d-1cf0de58597b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":17,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6821","nonce":140771},{"type":"v1","addr":"172.21.15.169:6823","nonce":140771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6825","nonce":140771},{"type":"v1","addr":"172.21.15.169:6826","nonce":140771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":140771},{"type":"v1","addr":"172.21.15.169:6831","nonce":140771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":140771},{"type":"v1","addr":"172.21.15.169:6829","nonce":140771}]},"public_addr":"172.21.15.169:6823/140771","cluster_addr":"172.21.15.169:6826/140771","heartbeat_back_addr":"172.21.15.169:6831/140771","heartbeat_front_addr":"172.21.15.169:6829/140771","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.821296+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.796648+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.816487+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.889969+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.819961+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.838840+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.817619+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.849885+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:08:11.708 INFO:tasks.ceph.ceph_manager.ceph:all up! 2023-09-22T13:08:11.708 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-09-22T13:08:11.996 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:08:11.996 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":18,"fsid":"583b11f8-631b-40f6-ac70-b90b0e6977fb","created":"2023-09-22T13:07:58.230141+0000","modified":"2023-09-22T13:08:11.021187+0000","last_up_change":"2023-09-22T13:08:03.695960+0000","last_in_change":"2023-09-22T13:08:00.680979+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-09-22T13:08:03.733606+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-22T13:08:09.395420+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"5482e19a-2a28-4455-8110-fb834d873a55","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":140877},{"type":"v1","addr":"172.21.15.98:6809","nonce":140877}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":140877},{"type":"v1","addr":"172.21.15.98:6811","nonce":140877}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":140877},{"type":"v1","addr":"172.21.15.98:6815","nonce":140877}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":140877},{"type":"v1","addr":"172.21.15.98:6813","nonce":140877}]},"public_addr":"172.21.15.98:6809/140877","cluster_addr":"172.21.15.98:6811/140877","heartbeat_back_addr":"172.21.15.98:6815/140877","heartbeat_front_addr":"172.21.15.98:6813/140877","state":["exists","up"]},{"osd":1,"uuid":"d9714f8e-eda9-40fe-a1bd-eaff19e52ba6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6800","nonce":140876},{"type":"v1","addr":"172.21.15.98:6801","nonce":140876}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":140876},{"type":"v1","addr":"172.21.15.98:6803","nonce":140876}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":140876},{"type":"v1","addr":"172.21.15.98:6807","nonce":140876}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":140876},{"type":"v1","addr":"172.21.15.98:6805","nonce":140876}]},"public_addr":"172.21.15.98:6801/140876","cluster_addr":"172.21.15.98:6803/140876","heartbeat_back_addr":"172.21.15.98:6807/140876","heartbeat_front_addr":"172.21.15.98:6805/140876","state":["exists","up"]},{"osd":2,"uuid":"f224f2a4-77ec-49e2-8a73-2a191cd83f27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":140892},{"type":"v1","addr":"172.21.15.98:6817","nonce":140892}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":140892},{"type":"v1","addr":"172.21.15.98:6819","nonce":140892}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":140892},{"type":"v1","addr":"172.21.15.98:6831","nonce":140892}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6821","nonce":140892},{"type":"v1","addr":"172.21.15.98:6829","nonce":140892}]},"public_addr":"172.21.15.98:6817/140892","cluster_addr":"172.21.15.98:6819/140892","heartbeat_back_addr":"172.21.15.98:6831/140892","heartbeat_front_addr":"172.21.15.98:6829/140892","state":["exists","up"]},{"osd":3,"uuid":"a43653eb-4a91-4b86-a125-e274d80ed338","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":17,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":140891},{"type":"v1","addr":"172.21.15.98:6822","nonce":140891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6823","nonce":140891},{"type":"v1","addr":"172.21.15.98:6824","nonce":140891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6827","nonce":140891},{"type":"v1","addr":"172.21.15.98:6828","nonce":140891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6825","nonce":140891},{"type":"v1","addr":"172.21.15.98:6826","nonce":140891}]},"public_addr":"172.21.15.98:6822/140891","cluster_addr":"172.21.15.98:6824/140891","heartbeat_back_addr":"172.21.15.98:6828/140891","heartbeat_front_addr":"172.21.15.98:6826/140891","state":["exists","up"]},{"osd":4,"uuid":"783c24fd-c283-436f-afd6-77320755247e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":17,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":140693},{"type":"v1","addr":"172.21.15.169:6801","nonce":140693}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":140693},{"type":"v1","addr":"172.21.15.169:6803","nonce":140693}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":140693},{"type":"v1","addr":"172.21.15.169:6807","nonce":140693}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":140693},{"type":"v1","addr":"172.21.15.169:6805","nonce":140693}]},"public_addr":"172.21.15.169:6801/140693","cluster_addr":"172.21.15.169:6803/140693","heartbeat_back_addr":"172.21.15.169:6807/140693","heartbeat_front_addr":"172.21.15.169:6805/140693","state":["exists","up"]},{"osd":5,"uuid":"e9410a8c-5c45-404a-b5e2-1686fe079232","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":17,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":140717},{"type":"v1","addr":"172.21.15.169:6817","nonce":140717}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":140717},{"type":"v1","addr":"172.21.15.169:6819","nonce":140717}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":140717},{"type":"v1","addr":"172.21.15.169:6827","nonce":140717}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":140717},{"type":"v1","addr":"172.21.15.169:6822","nonce":140717}]},"public_addr":"172.21.15.169:6817/140717","cluster_addr":"172.21.15.169:6819/140717","heartbeat_back_addr":"172.21.15.169:6827/140717","heartbeat_front_addr":"172.21.15.169:6822/140717","state":["exists","up"]},{"osd":6,"uuid":"0fe4f080-1273-4d09-a804-4fc9efb89815","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":140694},{"type":"v1","addr":"172.21.15.169:6809","nonce":140694}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":140694},{"type":"v1","addr":"172.21.15.169:6811","nonce":140694}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":140694},{"type":"v1","addr":"172.21.15.169:6815","nonce":140694}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":140694},{"type":"v1","addr":"172.21.15.169:6813","nonce":140694}]},"public_addr":"172.21.15.169:6809/140694","cluster_addr":"172.21.15.169:6811/140694","heartbeat_back_addr":"172.21.15.169:6815/140694","heartbeat_front_addr":"172.21.15.169:6813/140694","state":["exists","up"]},{"osd":7,"uuid":"f0dedb37-14f4-4e9e-ab7d-1cf0de58597b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":17,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6821","nonce":140771},{"type":"v1","addr":"172.21.15.169:6823","nonce":140771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6825","nonce":140771},{"type":"v1","addr":"172.21.15.169:6826","nonce":140771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":140771},{"type":"v1","addr":"172.21.15.169:6831","nonce":140771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":140771},{"type":"v1","addr":"172.21.15.169:6829","nonce":140771}]},"public_addr":"172.21.15.169:6823/140771","cluster_addr":"172.21.15.169:6826/140771","heartbeat_back_addr":"172.21.15.169:6831/140771","heartbeat_front_addr":"172.21.15.169:6829/140771","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.821296+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.796648+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.816487+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.889969+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.819961+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.838840+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.817619+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-09-22T13:08:01.849885+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-22T13:08:12.009 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2023-09-22T13:08:12.158 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:12.158 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2023-09-22T13:08:12.307 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:12.308 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2023-09-22T13:08:12.456 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:12.456 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2023-09-22T13:08:12.605 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:12.606 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.4 flush_pg_stats 2023-09-22T13:08:12.754 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:12.754 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.5 flush_pg_stats 2023-09-22T13:08:12.902 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:12.903 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.6 flush_pg_stats 2023-09-22T13:08:13.055 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:13.055 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.7 flush_pg_stats 2023-09-22T13:08:13.203 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:13.204 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2023-09-22T13:08:13.493 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:13.506 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574851 got 55834574851 for osd.0 2023-09-22T13:08:13.506 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2023-09-22T13:08:13.796 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:13.810 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574851 got 55834574851 for osd.1 2023-09-22T13:08:13.810 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2023-09-22T13:08:14.101 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:14.114 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574851 got 55834574851 for osd.2 2023-09-22T13:08:14.114 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2023-09-22T13:08:14.401 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:14.414 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574851 got 55834574851 for osd.3 2023-09-22T13:08:14.414 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.4 2023-09-22T13:08:14.701 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:14.714 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574851 got 55834574851 for osd.4 2023-09-22T13:08:14.715 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.5 2023-09-22T13:08:15.006 INFO:teuthology.orchestra.run.smithi098.stdout:55834574851 2023-09-22T13:08:15.019 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574851 got 55834574851 for osd.5 2023-09-22T13:08:15.019 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.6 2023-09-22T13:08:15.309 INFO:teuthology.orchestra.run.smithi098.stdout:55834574852 2023-09-22T13:08:15.323 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574851 got 55834574852 for osd.6 2023-09-22T13:08:15.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.7 2023-09-22T13:08:15.611 INFO:teuthology.orchestra.run.smithi098.stdout:55834574852 2023-09-22T13:08:15.624 INFO:tasks.ceph.ceph_manager.ceph:need seq 55834574851 got 55834574852 for osd.7 2023-09-22T13:08:15.624 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2023-09-22T13:08:15.625 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2023-09-22T13:08:15.910 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:08:15.912 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-22T13:08:15.922 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2023-09-22T13:08:14.930897+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":16,"num_osds":8,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":749595040,"kb_used":6367496,"kb_used_data":743227896,"kb_used_omap":329,"kb_used_meta":0,"kb_avail":743227544,"statfs":{"total":767585320960,"available":761065005056,"internally_reserved":360448,"allocated":761065365504,"data_stored":761065365504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":337508,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.339006+0000","last_change":"2023-09-22T13:08:11.338771+0000","last_active":"2023-09-22T13:08:11.339006+0000","last_peered":"2023-09-22T13:08:11.339006+0000","last_clean":"2023-09-22T13:08:11.339006+0000","last_became_active":"2023-09-22T13:08:11.338539+0000","last_became_peered":"2023-09-22T13:08:11.338539+0000","last_unstale":"2023-09-22T13:08:11.339006+0000","last_undegraded":"2023-09-22T13:08:11.339006+0000","last_fullsized":"2023-09-22T13:08:11.339006+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.339057+0000","last_change":"2023-09-22T13:08:11.338882+0000","last_active":"2023-09-22T13:08:11.339057+0000","last_peered":"2023-09-22T13:08:11.339057+0000","last_clean":"2023-09-22T13:08:11.339057+0000","last_became_active":"2023-09-22T13:08:11.338593+0000","last_became_peered":"2023-09-22T13:08:11.338593+0000","last_unstale":"2023-09-22T13:08:11.339057+0000","last_undegraded":"2023-09-22T13:08:11.339057+0000","last_fullsized":"2023-09-22T13:08:11.339057+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.036024+0000","last_change":"2023-09-22T13:08:11.035888+0000","last_active":"2023-09-22T13:08:11.036024+0000","last_peered":"2023-09-22T13:08:11.036024+0000","last_clean":"2023-09-22T13:08:11.036024+0000","last_became_active":"2023-09-22T13:08:11.035668+0000","last_became_peered":"2023-09-22T13:08:11.035668+0000","last_unstale":"2023-09-22T13:08:11.036024+0000","last_undegraded":"2023-09-22T13:08:11.036024+0000","last_fullsized":"2023-09-22T13:08:11.036024+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.035161+0000","last_change":"2023-09-22T13:08:11.035017+0000","last_active":"2023-09-22T13:08:11.035161+0000","last_peered":"2023-09-22T13:08:11.035161+0000","last_clean":"2023-09-22T13:08:11.035161+0000","last_became_active":"2023-09-22T13:08:11.034773+0000","last_became_peered":"2023-09-22T13:08:11.034773+0000","last_unstale":"2023-09-22T13:08:11.035161+0000","last_undegraded":"2023-09-22T13:08:11.035161+0000","last_fullsized":"2023-09-22T13:08:11.035161+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":12,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.337637+0000","last_change":"2023-09-22T13:08:05.720317+0000","last_active":"2023-09-22T13:08:11.337637+0000","last_peered":"2023-09-22T13:08:11.337637+0000","last_clean":"2023-09-22T13:08:11.337637+0000","last_became_active":"2023-09-22T13:08:05.720058+0000","last_became_peered":"2023-09-22T13:08:05.720058+0000","last_unstale":"2023-09-22T13:08:11.337637+0000","last_undegraded":"2023-09-22T13:08:11.337637+0000","last_fullsized":"2023-09-22T13:08:11.337637+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:04.700516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:04.700516+0000","last_clean_scrub_stamp":"2023-09-22T13:08:04.700516+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.338852+0000","last_change":"2023-09-22T13:08:11.338627+0000","last_active":"2023-09-22T13:08:11.338852+0000","last_peered":"2023-09-22T13:08:11.338852+0000","last_clean":"2023-09-22T13:08:11.338852+0000","last_became_active":"2023-09-22T13:08:11.338361+0000","last_became_peered":"2023-09-22T13:08:11.338361+0000","last_unstale":"2023-09-22T13:08:11.338852+0000","last_undegraded":"2023-09-22T13:08:11.338852+0000","last_fullsized":"2023-09-22T13:08:11.338852+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.032960+0000","last_change":"2023-09-22T13:08:11.032429+0000","last_active":"2023-09-22T13:08:11.032960+0000","last_peered":"2023-09-22T13:08:11.032960+0000","last_clean":"2023-09-22T13:08:11.032960+0000","last_became_active":"2023-09-22T13:08:11.032108+0000","last_became_peered":"2023-09-22T13:08:11.032108+0000","last_unstale":"2023-09-22T13:08:11.032960+0000","last_undegraded":"2023-09-22T13:08:11.032960+0000","last_fullsized":"2023-09-22T13:08:11.032960+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.035881+0000","last_change":"2023-09-22T13:08:11.035779+0000","last_active":"2023-09-22T13:08:11.035881+0000","last_peered":"2023-09-22T13:08:11.035881+0000","last_clean":"2023-09-22T13:08:11.035881+0000","last_became_active":"2023-09-22T13:08:11.035566+0000","last_became_peered":"2023-09-22T13:08:11.035566+0000","last_unstale":"2023-09-22T13:08:11.035881+0000","last_undegraded":"2023-09-22T13:08:11.035881+0000","last_fullsized":"2023-09-22T13:08:11.035881+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.034005+0000","last_change":"2023-09-22T13:08:11.033730+0000","last_active":"2023-09-22T13:08:11.034005+0000","last_peered":"2023-09-22T13:08:11.034005+0000","last_clean":"2023-09-22T13:08:11.034005+0000","last_became_active":"2023-09-22T13:08:11.033392+0000","last_became_peered":"2023-09-22T13:08:11.033392+0000","last_unstale":"2023-09-22T13:08:11.034005+0000","last_undegraded":"2023-09-22T13:08:11.034005+0000","last_fullsized":"2023-09-22T13:08:11.034005+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":0},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":2,"acting":2,"num_store_stats":0}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574852,"num_pgs":1,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795920,"kb_used_data":92903488,"kb_used_omap":33,"kb_used_meta":0,"kb_avail":92903460,"statfs":{"total":95948165120,"available":95133143040,"internally_reserved":28672,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34756,"internal_metadata":0},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40999999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47999999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52200000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61399999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70499999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73399999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72199999999999998}]}]},{"osd":6,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795940,"kb_used_data":92903488,"kb_used_omap":43,"kb_used_meta":0,"kb_avail":92903440,"statfs":{"total":95948165120,"available":95133122560,"internally_reserved":49152,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44220,"internal_metadata":0},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35599999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40000000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42199999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.93600000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53300000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59099999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574852,"num_pgs":1,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795908,"kb_used_data":92903488,"kb_used_omap":33,"kb_used_meta":0,"kb_avail":92903472,"statfs":{"total":95948165120,"available":95133155328,"internally_reserved":16384,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34756,"internal_metadata":0},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.251}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27600000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36699999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35699999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48999999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53100000000000003}]}]},{"osd":0,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795940,"kb_used_data":92903488,"kb_used_omap":38,"kb_used_meta":0,"kb_avail":92903440,"statfs":{"total":95948165120,"available":95133122560,"internally_reserved":49152,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39762,"internal_metadata":0},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.255}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32800000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38600000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54700000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63200000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52000000000000002}]}]},{"osd":2,"up_from":13,"seq":55834574852,"num_pgs":1,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795924,"kb_used_data":92903488,"kb_used_omap":38,"kb_used_meta":0,"kb_avail":92903456,"statfs":{"total":95948165120,"available":95133138944,"internally_reserved":32768,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39170,"internal_metadata":0},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32600000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35999999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64500000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65800000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53100000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.628}]}]},{"osd":3,"up_from":13,"seq":55834574852,"num_pgs":5,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795972,"kb_used_data":92903484,"kb_used_omap":53,"kb_used_meta":0,"kb_avail":92903408,"statfs":{"total":95948165120,"available":95133089792,"internally_reserved":77824,"allocated":95133167616,"data_stored":95133167616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55132,"internal_metadata":0},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.379}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39100000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56299999999999994}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.626}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65600000000000003}]}]},{"osd":4,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795936,"kb_used_data":92903488,"kb_used_omap":38,"kb_used_meta":0,"kb_avail":92903444,"statfs":{"total":95948165120,"available":95133126656,"internally_reserved":45056,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39806,"internal_metadata":0},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61899999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64000000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48899999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.5}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55700000000000005}]}]},{"osd":5,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795956,"kb_used_data":92903484,"kb_used_omap":48,"kb_used_meta":0,"kb_avail":92903424,"statfs":{"total":95948165120,"available":95133106176,"internally_reserved":61440,"allocated":95133167616,"data_stored":95133167616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49906,"internal_metadata":0},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65200000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.495}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59399999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55200000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66700000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67300000000000004}]}]}],"pool_statfs":[]}} 2023-09-22T13:08:15.925 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2023-09-22T13:08:16.209 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:08:16.211 INFO:teuthology.orchestra.run.smithi098.stderr:dumped all 2023-09-22T13:08:16.222 INFO:teuthology.orchestra.run.smithi098.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2023-09-22T13:08:14.930897+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":16,"num_osds":8,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":749595040,"kb_used":6367496,"kb_used_data":743227896,"kb_used_omap":329,"kb_used_meta":0,"kb_avail":743227544,"statfs":{"total":767585320960,"available":761065005056,"internally_reserved":360448,"allocated":761065365504,"data_stored":761065365504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":337508,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.339006+0000","last_change":"2023-09-22T13:08:11.338771+0000","last_active":"2023-09-22T13:08:11.339006+0000","last_peered":"2023-09-22T13:08:11.339006+0000","last_clean":"2023-09-22T13:08:11.339006+0000","last_became_active":"2023-09-22T13:08:11.338539+0000","last_became_peered":"2023-09-22T13:08:11.338539+0000","last_unstale":"2023-09-22T13:08:11.339006+0000","last_undegraded":"2023-09-22T13:08:11.339006+0000","last_fullsized":"2023-09-22T13:08:11.339006+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.339057+0000","last_change":"2023-09-22T13:08:11.338882+0000","last_active":"2023-09-22T13:08:11.339057+0000","last_peered":"2023-09-22T13:08:11.339057+0000","last_clean":"2023-09-22T13:08:11.339057+0000","last_became_active":"2023-09-22T13:08:11.338593+0000","last_became_peered":"2023-09-22T13:08:11.338593+0000","last_unstale":"2023-09-22T13:08:11.339057+0000","last_undegraded":"2023-09-22T13:08:11.339057+0000","last_fullsized":"2023-09-22T13:08:11.339057+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.036024+0000","last_change":"2023-09-22T13:08:11.035888+0000","last_active":"2023-09-22T13:08:11.036024+0000","last_peered":"2023-09-22T13:08:11.036024+0000","last_clean":"2023-09-22T13:08:11.036024+0000","last_became_active":"2023-09-22T13:08:11.035668+0000","last_became_peered":"2023-09-22T13:08:11.035668+0000","last_unstale":"2023-09-22T13:08:11.036024+0000","last_undegraded":"2023-09-22T13:08:11.036024+0000","last_fullsized":"2023-09-22T13:08:11.036024+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.035161+0000","last_change":"2023-09-22T13:08:11.035017+0000","last_active":"2023-09-22T13:08:11.035161+0000","last_peered":"2023-09-22T13:08:11.035161+0000","last_clean":"2023-09-22T13:08:11.035161+0000","last_became_active":"2023-09-22T13:08:11.034773+0000","last_became_peered":"2023-09-22T13:08:11.034773+0000","last_unstale":"2023-09-22T13:08:11.035161+0000","last_undegraded":"2023-09-22T13:08:11.035161+0000","last_fullsized":"2023-09-22T13:08:11.035161+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":12,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.337637+0000","last_change":"2023-09-22T13:08:05.720317+0000","last_active":"2023-09-22T13:08:11.337637+0000","last_peered":"2023-09-22T13:08:11.337637+0000","last_clean":"2023-09-22T13:08:11.337637+0000","last_became_active":"2023-09-22T13:08:05.720058+0000","last_became_peered":"2023-09-22T13:08:05.720058+0000","last_unstale":"2023-09-22T13:08:11.337637+0000","last_undegraded":"2023-09-22T13:08:11.337637+0000","last_fullsized":"2023-09-22T13:08:11.337637+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:04.700516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:04.700516+0000","last_clean_scrub_stamp":"2023-09-22T13:08:04.700516+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.338852+0000","last_change":"2023-09-22T13:08:11.338627+0000","last_active":"2023-09-22T13:08:11.338852+0000","last_peered":"2023-09-22T13:08:11.338852+0000","last_clean":"2023-09-22T13:08:11.338852+0000","last_became_active":"2023-09-22T13:08:11.338361+0000","last_became_peered":"2023-09-22T13:08:11.338361+0000","last_unstale":"2023-09-22T13:08:11.338852+0000","last_undegraded":"2023-09-22T13:08:11.338852+0000","last_fullsized":"2023-09-22T13:08:11.338852+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.032960+0000","last_change":"2023-09-22T13:08:11.032429+0000","last_active":"2023-09-22T13:08:11.032960+0000","last_peered":"2023-09-22T13:08:11.032960+0000","last_clean":"2023-09-22T13:08:11.032960+0000","last_became_active":"2023-09-22T13:08:11.032108+0000","last_became_peered":"2023-09-22T13:08:11.032108+0000","last_unstale":"2023-09-22T13:08:11.032960+0000","last_undegraded":"2023-09-22T13:08:11.032960+0000","last_fullsized":"2023-09-22T13:08:11.032960+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.035881+0000","last_change":"2023-09-22T13:08:11.035779+0000","last_active":"2023-09-22T13:08:11.035881+0000","last_peered":"2023-09-22T13:08:11.035881+0000","last_clean":"2023-09-22T13:08:11.035881+0000","last_became_active":"2023-09-22T13:08:11.035566+0000","last_became_peered":"2023-09-22T13:08:11.035566+0000","last_unstale":"2023-09-22T13:08:11.035881+0000","last_undegraded":"2023-09-22T13:08:11.035881+0000","last_fullsized":"2023-09-22T13:08:11.035881+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":9,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-09-22T13:08:11.034005+0000","last_change":"2023-09-22T13:08:11.033730+0000","last_active":"2023-09-22T13:08:11.034005+0000","last_peered":"2023-09-22T13:08:11.034005+0000","last_clean":"2023-09-22T13:08:11.034005+0000","last_became_active":"2023-09-22T13:08:11.033392+0000","last_became_peered":"2023-09-22T13:08:11.033392+0000","last_unstale":"2023-09-22T13:08:11.034005+0000","last_undegraded":"2023-09-22T13:08:11.034005+0000","last_fullsized":"2023-09-22T13:08:11.034005+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-22T13:08:10.010922+0000","last_clean_scrub_stamp":"2023-09-22T13:08:10.010922+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":0},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":2,"acting":2,"num_store_stats":0}],"osd_stats":[{"osd":7,"up_from":13,"seq":55834574852,"num_pgs":1,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795920,"kb_used_data":92903488,"kb_used_omap":33,"kb_used_meta":0,"kb_avail":92903460,"statfs":{"total":95948165120,"available":95133143040,"internally_reserved":28672,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34756,"internal_metadata":0},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40999999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47999999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52200000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61399999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70499999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73399999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72199999999999998}]}]},{"osd":6,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795940,"kb_used_data":92903488,"kb_used_omap":43,"kb_used_meta":0,"kb_avail":92903440,"statfs":{"total":95948165120,"available":95133122560,"internally_reserved":49152,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44220,"internal_metadata":0},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35599999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40000000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42199999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.93600000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53300000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59099999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574852,"num_pgs":1,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795908,"kb_used_data":92903488,"kb_used_omap":33,"kb_used_meta":0,"kb_avail":92903472,"statfs":{"total":95948165120,"available":95133155328,"internally_reserved":16384,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34756,"internal_metadata":0},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.251}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27600000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36699999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35699999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48999999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53100000000000003}]}]},{"osd":0,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795940,"kb_used_data":92903488,"kb_used_omap":38,"kb_used_meta":0,"kb_avail":92903440,"statfs":{"total":95948165120,"available":95133122560,"internally_reserved":49152,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39762,"internal_metadata":0},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.255}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32800000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38600000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54700000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63200000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52000000000000002}]}]},{"osd":2,"up_from":13,"seq":55834574852,"num_pgs":1,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795924,"kb_used_data":92903488,"kb_used_omap":38,"kb_used_meta":0,"kb_avail":92903456,"statfs":{"total":95948165120,"available":95133138944,"internally_reserved":32768,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39170,"internal_metadata":0},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32600000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35999999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64500000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65800000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53100000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.628}]}]},{"osd":3,"up_from":13,"seq":55834574852,"num_pgs":5,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795972,"kb_used_data":92903484,"kb_used_omap":53,"kb_used_meta":0,"kb_avail":92903408,"statfs":{"total":95948165120,"available":95133089792,"internally_reserved":77824,"allocated":95133167616,"data_stored":95133167616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55132,"internal_metadata":0},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.379}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39100000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56299999999999994}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.626}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65600000000000003}]}]},{"osd":4,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795936,"kb_used_data":92903488,"kb_used_omap":38,"kb_used_meta":0,"kb_avail":92903444,"statfs":{"total":95948165120,"available":95133126656,"internally_reserved":45056,"allocated":95133171712,"data_stored":95133171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39806,"internal_metadata":0},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61899999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64000000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48899999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41499999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.5}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55700000000000005}]}]},{"osd":5,"up_from":13,"seq":55834574852,"num_pgs":2,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93699380,"kb_used":795956,"kb_used_data":92903484,"kb_used_omap":48,"kb_used_meta":0,"kb_avail":92903424,"statfs":{"total":95948165120,"available":95133106176,"internally_reserved":61440,"allocated":95133167616,"data_stored":95133167616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49906,"internal_metadata":0},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65200000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.495}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59399999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55200000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66700000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67300000000000004}]}]}],"pool_statfs":[]}} 2023-09-22T13:08:16.223 INFO:tasks.ceph.ceph_manager.ceph:clean! 2023-09-22T13:08:16.224 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-22T13:08:16.224 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2023-09-22T13:08:16.224 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2023-09-22T13:08:16.545 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:08:16.558 INFO:teuthology.orchestra.run.smithi098.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-22T13:08:16.558 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2023-09-22T13:08:16.558 INFO:teuthology.run_tasks:Running task openssl_keys... 2023-09-22T13:08:16.571 INFO:teuthology.run_tasks:Running task rgw... 2023-09-22T13:08:16.607 DEBUG:tasks.rgw:config is {'client.0': None} 2023-09-22T13:08:16.607 DEBUG:tasks.rgw:client list is dict_keys(['client.0']) 2023-09-22T13:08:16.607 INFO:tasks.rgw:Creating data pools 2023-09-22T13:08:16.607 DEBUG:tasks.rgw:Obtaining remote for client client.0 2023-09-22T13:08:16.607 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph osd erasure-code-profile set client.0 crush-failure-domain=osd k=3 m=1 name=testprofile --cluster ceph 2023-09-22T13:08:17.065 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph osd pool create default.rgw.buckets.data 64 64 erasure client.0 --cluster ceph 2023-09-22T13:08:19.085 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'default.rgw.buckets.data' created 2023-09-22T13:08:19.098 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster ceph 2023-09-22T13:08:20.087 INFO:teuthology.orchestra.run.smithi098.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2023-09-22T13:08:20.106 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster ceph 2023-09-22T13:08:21.089 INFO:teuthology.orchestra.run.smithi098.stderr:pool 'default.rgw.buckets.index' created 2023-09-22T13:08:21.097 DEBUG:teuthology.orchestra.run.smithi098:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster ceph 2023-09-22T13:08:22.100 INFO:teuthology.orchestra.run.smithi098.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2023-09-22T13:08:22.112 DEBUG:tasks.rgw:Pools created 2023-09-22T13:08:22.113 INFO:tasks.rgw:Configuring compression type = random 2023-09-22T13:08:22.113 INFO:tasks.util.rgw:rgwadmin: client.0 : ['user', 'list'] 2023-09-22T13:08:22.113 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'user', 'list'] 2023-09-22T13:08:22.113 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph user list 2023-09-22T13:08:22.223 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:08:22.223 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:08:22.229 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:22.230+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.108 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.108+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.108 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.108+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.108 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.109+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.108 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.109+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.109 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.109+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.109 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.109+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.109 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.110+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.150 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.150+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.151 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.150+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.151 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.150+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.151 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.151+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.151 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.151+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.152 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.151+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.152 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.151+0000 7f825a093500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:08:24.152 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.151+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.152 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.151+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.152 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.151+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.152 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.152+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.152 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.152+0000 7f825a093500 10 Creating default zonegroup 2023-09-22T13:08:24.152 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.152+0000 7f825a093500 20 get_system_obj_state: rctx=0x7fff664eb4b0 obj=:.pools.avail state=0x562a57745810 s->prefetch_data=0 2023-09-22T13:08:24.153 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.152+0000 7f825a093500 10 couldn't find old data placement pools config, setting up new ones for the zone 2023-09-22T13:08:24.156 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.157+0000 7f8244d74700 0 -- 172.21.15.98:0/333490916 >> [v2:172.21.15.169:6808/140694,v1:172.21.15.169:6809/140694] conn(0x562a578823e0 msgr2=0x562a578847d0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:08:24.161 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.161+0000 7f825a093500 10 WARNING: store->list_zones() returned r=-2 2023-09-22T13:08:24.161 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.161+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.161 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.161+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.162 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.162+0000 7f8244d74700 0 -- 172.21.15.98:0/333490916 >> [v2:172.21.15.98:6800/140876,v1:172.21.15.98:6801/140876] conn(0x562a578895f0 msgr2=0x562a5788b9e0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:08:24.163 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.163+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.163 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.163+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.164 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.163+0000 7f825a093500 10 could not read realm id: (2) No such file or directory 2023-09-22T13:08:24.164 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.163+0000 7f825a093500 10 WARNING: failed to set zone as default, r=-22 2023-09-22T13:08:24.164 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.163+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.164 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.164+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.165 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.165+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.166+0000 7f825a093500 20 rados_obj.operate() r=0 bl.length=873 2023-09-22T13:08:24.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.166+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.166+0000 7f825a093500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:08:24.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.166+0000 7f825a093500 20 zonegroup default 2023-09-22T13:08:24.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.166+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.166+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:24.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.166+0000 7f825a093500 10 Cannot find current period zone using local zone 2023-09-22T13:08:24.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.166+0000 7f825a093500 10 Using default name default 2023-09-22T13:08:24.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.166+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.166+0000 7f825a093500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:24.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.166+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:24.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.167+0000 7f825a093500 20 rados_obj.operate() r=0 bl.length=873 2023-09-22T13:08:24.168 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.167+0000 7f825a093500 20 zone default found 2023-09-22T13:08:24.168 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:24.167+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:26.142 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:26.141+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:26.142 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:26.141+0000 7f825a093500 20 rados->read ofs=0 len=0 2023-09-22T13:08:26.143 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:26.142+0000 7f825a093500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:26.143 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:26.142+0000 7f825a093500 20 started sync module instance, tier type = 2023-09-22T13:08:26.143 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:26.142+0000 7f825a093500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:08:28.155 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.154+0000 7f825a093500 20 add_watcher() i=0 2023-09-22T13:08:28.155 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.154+0000 7f825a093500 20 add_watcher() i=1 2023-09-22T13:08:28.155 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.154+0000 7f825a093500 20 add_watcher() i=2 2023-09-22T13:08:28.155 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.154+0000 7f825a093500 20 add_watcher() i=3 2023-09-22T13:08:28.156 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.154+0000 7f825a093500 20 add_watcher() i=4 2023-09-22T13:08:28.156 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.154+0000 7f825a093500 20 add_watcher() i=5 2023-09-22T13:08:28.156 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.154+0000 7f825a093500 20 add_watcher() i=6 2023-09-22T13:08:28.163 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.154+0000 7f825a093500 20 add_watcher() i=7 2023-09-22T13:08:28.163 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.154+0000 7f825a093500 2 all 8 watchers are set, enabling cache 2023-09-22T13:08:28.163 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.155+0000 7f825a093500 5 tl::expected, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):409: oid=data_loggenerations_metadata not found 2023-09-22T13:08:28.163 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.155+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.0 2023-09-22T13:08:28.164 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.155+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.164 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.155+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.0 does not exist 2023-09-22T13:08:28.164 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.155+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.1 2023-09-22T13:08:28.164 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.155+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.164 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.155+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.1 does not exist 2023-09-22T13:08:28.164 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.155+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.2 2023-09-22T13:08:28.164 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.155+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.165 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.2 does not exist 2023-09-22T13:08:28.165 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.3 2023-09-22T13:08:28.165 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.165 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.3 does not exist 2023-09-22T13:08:28.165 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.4 2023-09-22T13:08:28.165 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.165 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.4 does not exist 2023-09-22T13:08:28.165 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.5 2023-09-22T13:08:28.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.5 does not exist 2023-09-22T13:08:28.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.6 2023-09-22T13:08:28.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.156+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.6 does not exist 2023-09-22T13:08:28.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.7 2023-09-22T13:08:28.166 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.7 does not exist 2023-09-22T13:08:28.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.8 2023-09-22T13:08:28.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.8 does not exist 2023-09-22T13:08:28.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.9 2023-09-22T13:08:28.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.167 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.9 does not exist 2023-09-22T13:08:28.168 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.10 2023-09-22T13:08:28.168 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.168 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.10 does not exist 2023-09-22T13:08:28.168 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.11 2023-09-22T13:08:28.168 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.157+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.168 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.11 does not exist 2023-09-22T13:08:28.168 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.12 2023-09-22T13:08:28.168 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.169 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.12 does not exist 2023-09-22T13:08:28.169 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.13 2023-09-22T13:08:28.169 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.169 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.13 does not exist 2023-09-22T13:08:28.169 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.14 2023-09-22T13:08:28.169 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.169 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.14 does not exist 2023-09-22T13:08:28.169 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.15 2023-09-22T13:08:28.170 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.158+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.170 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.15 does not exist 2023-09-22T13:08:28.170 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.16 2023-09-22T13:08:28.170 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.170 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.16 does not exist 2023-09-22T13:08:28.170 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.17 2023-09-22T13:08:28.170 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.170 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.17 does not exist 2023-09-22T13:08:28.171 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.18 2023-09-22T13:08:28.171 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.171 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.18 does not exist 2023-09-22T13:08:28.171 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.19 2023-09-22T13:08:28.171 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.159+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.171 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.19 does not exist 2023-09-22T13:08:28.171 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.20 2023-09-22T13:08:28.171 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.172 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.20 does not exist 2023-09-22T13:08:28.172 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.21 2023-09-22T13:08:28.172 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.172 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.21 does not exist 2023-09-22T13:08:28.172 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.22 2023-09-22T13:08:28.172 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.172 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.22 does not exist 2023-09-22T13:08:28.172 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.23 2023-09-22T13:08:28.173 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.160+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.173 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.23 does not exist 2023-09-22T13:08:28.173 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.24 2023-09-22T13:08:28.173 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.173 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.24 does not exist 2023-09-22T13:08:28.173 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.25 2023-09-22T13:08:28.173 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.174 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.25 does not exist 2023-09-22T13:08:28.174 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.26 2023-09-22T13:08:28.174 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.174 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.26 does not exist 2023-09-22T13:08:28.174 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.27 2023-09-22T13:08:28.174 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.161+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.174 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.27 does not exist 2023-09-22T13:08:28.174 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.28 2023-09-22T13:08:28.175 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.175 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.28 does not exist 2023-09-22T13:08:28.175 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.29 2023-09-22T13:08:28.175 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.175 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.29 does not exist 2023-09-22T13:08:28.175 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.30 2023-09-22T13:08:28.175 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.175 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.30 does not exist 2023-09-22T13:08:28.176 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.31 2023-09-22T13:08:28.176 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.162+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.192 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.31 does not exist 2023-09-22T13:08:28.192 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.32 2023-09-22T13:08:28.192 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.192 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.32 does not exist 2023-09-22T13:08:28.192 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.33 2023-09-22T13:08:28.192 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.193 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.33 does not exist 2023-09-22T13:08:28.193 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.34 2023-09-22T13:08:28.193 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.193 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.34 does not exist 2023-09-22T13:08:28.193 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.35 2023-09-22T13:08:28.193 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.163+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.193 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.35 does not exist 2023-09-22T13:08:28.193 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.36 2023-09-22T13:08:28.194 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.194 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.36 does not exist 2023-09-22T13:08:28.194 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.37 2023-09-22T13:08:28.194 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.194 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.37 does not exist 2023-09-22T13:08:28.194 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.38 2023-09-22T13:08:28.194 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.194 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.38 does not exist 2023-09-22T13:08:28.195 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.39 2023-09-22T13:08:28.195 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.164+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.195 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.39 does not exist 2023-09-22T13:08:28.195 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.40 2023-09-22T13:08:28.195 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.195 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.40 does not exist 2023-09-22T13:08:28.195 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.41 2023-09-22T13:08:28.195 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.196 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.41 does not exist 2023-09-22T13:08:28.196 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.42 2023-09-22T13:08:28.196 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.196 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.42 does not exist 2023-09-22T13:08:28.196 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.43 2023-09-22T13:08:28.196 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.165+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.196 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.43 does not exist 2023-09-22T13:08:28.196 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.44 2023-09-22T13:08:28.196 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.197 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.44 does not exist 2023-09-22T13:08:28.197 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.45 2023-09-22T13:08:28.197 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.197 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.45 does not exist 2023-09-22T13:08:28.197 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.46 2023-09-22T13:08:28.197 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.197 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.46 does not exist 2023-09-22T13:08:28.197 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.47 2023-09-22T13:08:28.197 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.166+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.198 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.167+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.47 does not exist 2023-09-22T13:08:28.198 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.167+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.48 2023-09-22T13:08:28.198 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.167+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.198 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.167+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.48 does not exist 2023-09-22T13:08:28.198 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.167+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.49 2023-09-22T13:08:28.198 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.167+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.198 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.167+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.49 does not exist 2023-09-22T13:08:28.198 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.167+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.50 2023-09-22T13:08:28.198 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.167+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.199 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.50 does not exist 2023-09-22T13:08:28.199 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.51 2023-09-22T13:08:28.199 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.199 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.51 does not exist 2023-09-22T13:08:28.199 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.52 2023-09-22T13:08:28.199 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.199 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.52 does not exist 2023-09-22T13:08:28.199 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.53 2023-09-22T13:08:28.199 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.199 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.53 does not exist 2023-09-22T13:08:28.200 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.54 2023-09-22T13:08:28.200 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.168+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.200 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.54 does not exist 2023-09-22T13:08:28.200 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.55 2023-09-22T13:08:28.200 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.200 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.55 does not exist 2023-09-22T13:08:28.200 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.56 2023-09-22T13:08:28.200 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.201 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.56 does not exist 2023-09-22T13:08:28.201 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.57 2023-09-22T13:08:28.201 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.201 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.57 does not exist 2023-09-22T13:08:28.201 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.58 2023-09-22T13:08:28.201 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.169+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.201 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.58 does not exist 2023-09-22T13:08:28.201 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.59 2023-09-22T13:08:28.201 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.201 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.59 does not exist 2023-09-22T13:08:28.202 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.60 2023-09-22T13:08:28.202 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.202 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.60 does not exist 2023-09-22T13:08:28.202 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.61 2023-09-22T13:08:28.202 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.202 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.61 does not exist 2023-09-22T13:08:28.202 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.62 2023-09-22T13:08:28.202 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.170+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.202 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.62 does not exist 2023-09-22T13:08:28.203 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.63 2023-09-22T13:08:28.203 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.203 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.63 does not exist 2023-09-22T13:08:28.203 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.64 2023-09-22T13:08:28.203 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.203 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.64 does not exist 2023-09-22T13:08:28.203 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.65 2023-09-22T13:08:28.203 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.203 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.65 does not exist 2023-09-22T13:08:28.203 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.66 2023-09-22T13:08:28.204 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.171+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.204 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.66 does not exist 2023-09-22T13:08:28.204 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.67 2023-09-22T13:08:28.204 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.204 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.67 does not exist 2023-09-22T13:08:28.204 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.68 2023-09-22T13:08:28.204 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.204 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.68 does not exist 2023-09-22T13:08:28.205 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.69 2023-09-22T13:08:28.205 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.205 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.69 does not exist 2023-09-22T13:08:28.205 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.70 2023-09-22T13:08:28.205 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.205 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.70 does not exist 2023-09-22T13:08:28.205 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.71 2023-09-22T13:08:28.205 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.172+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.205 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.71 does not exist 2023-09-22T13:08:28.205 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.72 2023-09-22T13:08:28.206 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.206 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.72 does not exist 2023-09-22T13:08:28.206 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.73 2023-09-22T13:08:28.206 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.206 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.73 does not exist 2023-09-22T13:08:28.206 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.74 2023-09-22T13:08:28.206 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.206 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.74 does not exist 2023-09-22T13:08:28.206 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.75 2023-09-22T13:08:28.207 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.173+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.207 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.174+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.75 does not exist 2023-09-22T13:08:28.207 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.174+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.76 2023-09-22T13:08:28.207 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.174+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.207 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.174+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.76 does not exist 2023-09-22T13:08:28.207 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.174+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.77 2023-09-22T13:08:28.207 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.174+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.207 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.174+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.77 does not exist 2023-09-22T13:08:28.207 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.174+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.78 2023-09-22T13:08:28.207 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.174+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.208 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.78 does not exist 2023-09-22T13:08:28.208 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.79 2023-09-22T13:08:28.208 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.208 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.79 does not exist 2023-09-22T13:08:28.208 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.80 2023-09-22T13:08:28.208 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.208 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.80 does not exist 2023-09-22T13:08:28.208 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.81 2023-09-22T13:08:28.208 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.209 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.81 does not exist 2023-09-22T13:08:28.209 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.82 2023-09-22T13:08:28.209 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.175+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.209 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.82 does not exist 2023-09-22T13:08:28.209 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.83 2023-09-22T13:08:28.209 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.209 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.83 does not exist 2023-09-22T13:08:28.209 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.84 2023-09-22T13:08:28.209 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.209 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.84 does not exist 2023-09-22T13:08:28.210 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.85 2023-09-22T13:08:28.210 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.210 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.85 does not exist 2023-09-22T13:08:28.210 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.86 2023-09-22T13:08:28.210 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.176+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.210 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.86 does not exist 2023-09-22T13:08:28.210 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.87 2023-09-22T13:08:28.210 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.210 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.87 does not exist 2023-09-22T13:08:28.211 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.88 2023-09-22T13:08:28.211 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.211 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.88 does not exist 2023-09-22T13:08:28.211 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.89 2023-09-22T13:08:28.211 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.211 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.89 does not exist 2023-09-22T13:08:28.211 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.90 2023-09-22T13:08:28.211 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.211 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.90 does not exist 2023-09-22T13:08:28.211 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.91 2023-09-22T13:08:28.212 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.177+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.212 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.91 does not exist 2023-09-22T13:08:28.212 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.92 2023-09-22T13:08:28.212 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.212 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.92 does not exist 2023-09-22T13:08:28.212 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.93 2023-09-22T13:08:28.212 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.212 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.93 does not exist 2023-09-22T13:08:28.212 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.94 2023-09-22T13:08:28.212 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.213 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.94 does not exist 2023-09-22T13:08:28.213 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.95 2023-09-22T13:08:28.213 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.178+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.213 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.95 does not exist 2023-09-22T13:08:28.213 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.96 2023-09-22T13:08:28.213 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.213 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.96 does not exist 2023-09-22T13:08:28.213 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.97 2023-09-22T13:08:28.213 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.214 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.97 does not exist 2023-09-22T13:08:28.214 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.98 2023-09-22T13:08:28.214 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.214 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.98 does not exist 2023-09-22T13:08:28.214 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.99 2023-09-22T13:08:28.214 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.179+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.214 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.99 does not exist 2023-09-22T13:08:28.214 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.100 2023-09-22T13:08:28.214 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.215 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.100 does not exist 2023-09-22T13:08:28.215 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.101 2023-09-22T13:08:28.215 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.215 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.101 does not exist 2023-09-22T13:08:28.215 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.102 2023-09-22T13:08:28.215 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.215 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.102 does not exist 2023-09-22T13:08:28.215 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.103 2023-09-22T13:08:28.215 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.216 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.103 does not exist 2023-09-22T13:08:28.216 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.104 2023-09-22T13:08:28.216 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.180+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.216 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.104 does not exist 2023-09-22T13:08:28.216 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.105 2023-09-22T13:08:28.216 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.216 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.105 does not exist 2023-09-22T13:08:28.216 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.106 2023-09-22T13:08:28.216 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.216 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.106 does not exist 2023-09-22T13:08:28.217 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.107 2023-09-22T13:08:28.217 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.217 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.107 does not exist 2023-09-22T13:08:28.217 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.108 2023-09-22T13:08:28.217 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.181+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.217 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.182+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.108 does not exist 2023-09-22T13:08:28.217 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.182+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.109 2023-09-22T13:08:28.217 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.182+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.217 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.182+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.109 does not exist 2023-09-22T13:08:28.217 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.182+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.110 2023-09-22T13:08:28.218 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.182+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.218 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.110 does not exist 2023-09-22T13:08:28.218 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.111 2023-09-22T13:08:28.218 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.218 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.111 does not exist 2023-09-22T13:08:28.218 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.112 2023-09-22T13:08:28.218 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.218 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.112 does not exist 2023-09-22T13:08:28.218 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.113 2023-09-22T13:08:28.219 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.219 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.113 does not exist 2023-09-22T13:08:28.219 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.114 2023-09-22T13:08:28.219 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.219 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.114 does not exist 2023-09-22T13:08:28.219 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.115 2023-09-22T13:08:28.219 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.183+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.219 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.115 does not exist 2023-09-22T13:08:28.219 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.116 2023-09-22T13:08:28.219 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.220 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.116 does not exist 2023-09-22T13:08:28.220 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.117 2023-09-22T13:08:28.220 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.220 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.117 does not exist 2023-09-22T13:08:28.220 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.118 2023-09-22T13:08:28.220 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.220 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.118 does not exist 2023-09-22T13:08:28.220 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.119 2023-09-22T13:08:28.220 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.184+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.221 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.119 does not exist 2023-09-22T13:08:28.221 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.120 2023-09-22T13:08:28.221 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.221 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.120 does not exist 2023-09-22T13:08:28.221 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.121 2023-09-22T13:08:28.221 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.221 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.121 does not exist 2023-09-22T13:08:28.221 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.122 2023-09-22T13:08:28.221 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.221 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.122 does not exist 2023-09-22T13:08:28.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.123 2023-09-22T13:08:28.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.185+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.123 does not exist 2023-09-22T13:08:28.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.124 2023-09-22T13:08:28.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.124 does not exist 2023-09-22T13:08:28.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.125 2023-09-22T13:08:28.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.125 does not exist 2023-09-22T13:08:28.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.126 2023-09-22T13:08:28.223 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.223 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.126 does not exist 2023-09-22T13:08:28.223 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):37 probing oid=data_log.127 2023-09-22T13:08:28.223 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.223 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):59: oid=data_log.127 does not exist 2023-09-22T13:08:28.223 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.186+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::create(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, std::optional >, bool, uint64_t, uint64_t):1206 entering 2023-09-22T13:08:28.223 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.187+0000 7f825a093500 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1164 entering 2023-09-22T13:08:28.223 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.189+0000 7f82167e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:08:28.223 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.190+0000 7f825a093500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:08:28.224 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:28.190+0000 7f825a093500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:08:31.122 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.121+0000 7f825a093500 10 rgw_init_ioctx warning: failed to set recovery_priority on default.rgw.meta 2023-09-22T13:08:31.122 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.121+0000 7f825a093500 5 note: GC not initialized 2023-09-22T13:08:31.125 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.123+0000 7f825a093500 20 init_complete bucket index max shards: 1 2023-09-22T13:08:31.125 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.123+0000 7f825a093500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:08:31.125 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.123+0000 7f81857fa700 20 reqs_thread_entry: start 2023-09-22T13:08:31.125 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.123+0000 7f817affd700 20 reqs_thread_entry: start 2023-09-22T13:08:31.125 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.123+0000 7f81797fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:09:01 2023 2023-09-22T13:08:31.125 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-22T13:08:31.125 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.124+0000 7f8178ff9700 20 reqs_thread_entry: start 2023-09-22T13:08:31.149 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.148+0000 7f825a093500 20 remove_watcher() i=0 2023-09-22T13:08:31.149 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.148+0000 7f825a093500 2 removed watcher, disabling cache 2023-09-22T13:08:31.149 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.149+0000 7f825a093500 20 remove_watcher() i=1 2023-09-22T13:08:31.150 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.150+0000 7f825a093500 20 remove_watcher() i=2 2023-09-22T13:08:31.151 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.150+0000 7f825a093500 20 remove_watcher() i=3 2023-09-22T13:08:31.152 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.151+0000 7f825a093500 20 remove_watcher() i=4 2023-09-22T13:08:31.153 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.152+0000 7f825a093500 20 remove_watcher() i=5 2023-09-22T13:08:31.154 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.153+0000 7f825a093500 20 remove_watcher() i=6 2023-09-22T13:08:31.155 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.154+0000 7f825a093500 20 remove_watcher() i=7 2023-09-22T13:08:31.156 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.155+0000 7f82167e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:08:31.159 INFO:teuthology.orchestra.run.smithi098.stdout:[] 2023-09-22T13:08:31.162 DEBUG:tasks.util.rgw: json result: [] 2023-09-22T13:08:31.162 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2023-09-22T13:08:31.162 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2023-09-22T13:08:31.163 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zone placement modify --rgw-zone default --placement-id default-placement --compression random 2023-09-22T13:08:31.211 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:08:31.211 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:08:31.218 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.217+0000 7f305ae57500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:08:31.218 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.217+0000 7f305ae57500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:08:31.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.221+0000 7f2f8ffff700 20 reqs_thread_entry: start 2023-09-22T13:08:31.222 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.221+0000 7f305ae57500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.223 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.222+0000 7f305ae57500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.223 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.222+0000 7f305ae57500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.224 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.223+0000 7f305ae57500 20 rados_obj.operate() r=0 bl.length=873 2023-09-22T13:08:31.224 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.223+0000 7f305ae57500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.225 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.224+0000 7f305ae57500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.225 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.224+0000 7f305ae57500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.225 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.224+0000 7f305ae57500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.225 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.224+0000 7f305ae57500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.226 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.225+0000 7f305ae57500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:08:31.229 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-22T13:08:31.229 INFO:teuthology.orchestra.run.smithi098.stdout: "id": "8b3505f7-cd1d-403d-9eb6-cc0558e0de01", 2023-09-22T13:08:31.229 INFO:teuthology.orchestra.run.smithi098.stdout: "name": "default", 2023-09-22T13:08:31.230 INFO:teuthology.orchestra.run.smithi098.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:08:31.230 INFO:teuthology.orchestra.run.smithi098.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:08:31.230 INFO:teuthology.orchestra.run.smithi098.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:08:31.230 INFO:teuthology.orchestra.run.smithi098.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:08:31.230 INFO:teuthology.orchestra.run.smithi098.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:08:31.230 INFO:teuthology.orchestra.run.smithi098.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:08:31.230 INFO:teuthology.orchestra.run.smithi098.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:08:31.231 INFO:teuthology.orchestra.run.smithi098.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:08:31.231 INFO:teuthology.orchestra.run.smithi098.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:08:31.231 INFO:teuthology.orchestra.run.smithi098.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:08:31.231 INFO:teuthology.orchestra.run.smithi098.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:08:31.231 INFO:teuthology.orchestra.run.smithi098.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:08:31.231 INFO:teuthology.orchestra.run.smithi098.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:08:31.231 INFO:teuthology.orchestra.run.smithi098.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:08:31.231 INFO:teuthology.orchestra.run.smithi098.stdout: "system_key": { 2023-09-22T13:08:31.232 INFO:teuthology.orchestra.run.smithi098.stdout: "access_key": "", 2023-09-22T13:08:31.232 INFO:teuthology.orchestra.run.smithi098.stdout: "secret_key": "" 2023-09-22T13:08:31.232 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:08:31.232 INFO:teuthology.orchestra.run.smithi098.stdout: "placement_pools": [ 2023-09-22T13:08:31.232 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-22T13:08:31.232 INFO:teuthology.orchestra.run.smithi098.stdout: "key": "default-placement", 2023-09-22T13:08:31.232 INFO:teuthology.orchestra.run.smithi098.stdout: "val": { 2023-09-22T13:08:31.232 INFO:teuthology.orchestra.run.smithi098.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:08:31.232 INFO:teuthology.orchestra.run.smithi098.stdout: "storage_classes": { 2023-09-22T13:08:31.233 INFO:teuthology.orchestra.run.smithi098.stdout: "STANDARD": { 2023-09-22T13:08:31.233 INFO:teuthology.orchestra.run.smithi098.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:08:31.233 INFO:teuthology.orchestra.run.smithi098.stdout: "compression_type": "random" 2023-09-22T13:08:31.233 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.233 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:08:31.233 INFO:teuthology.orchestra.run.smithi098.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:08:31.233 INFO:teuthology.orchestra.run.smithi098.stdout: "index_type": 0 2023-09-22T13:08:31.233 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.234 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.234 INFO:teuthology.orchestra.run.smithi098.stdout: ], 2023-09-22T13:08:31.234 INFO:teuthology.orchestra.run.smithi098.stdout: "realm_id": "", 2023-09-22T13:08:31.234 INFO:teuthology.orchestra.run.smithi098.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:08:31.234 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-22T13:08:31.235 DEBUG:tasks.util.rgw: json result: {'id': '8b3505f7-cd1d-403d-9eb6-cc0558e0de01', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:08:31.236 INFO:tasks.util.rgw:rgwadmin: client.0 : ['user', 'list'] 2023-09-22T13:08:31.236 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'user', 'list'] 2023-09-22T13:08:31.236 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph user list 2023-09-22T13:08:31.276 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:08:31.276 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:08:31.282 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.281+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.284 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.283+0000 7f2e0f985500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.284 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.283+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.285 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.284+0000 7f2e0f985500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.285 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.284+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.285 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.284+0000 7f2e0f985500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.285 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.284+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.285 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.284+0000 7f2e0f985500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.296 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.295+0000 7f2e0f985500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:08:31.297 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.295+0000 7f2e0f985500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:08:31.297 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.295+0000 7f2e0f985500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:08:31.297 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.295+0000 7f2e0f985500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:08:31.297 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.295+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.297 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.295+0000 7f2e0f985500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.297 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.295+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.297 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.295+0000 7f2e0f985500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:08:31.297 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.295+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.298 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.296+0000 7f2e0f985500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.298 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.296+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.300 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.296+0000 7f2e0f985500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:08:31.300 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.296+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.300 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.296+0000 7f2e0f985500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.300 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.296+0000 7f2e0f985500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:08:31.300 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.296+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.301 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.297+0000 7f2e0f985500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.301 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.297+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.301 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.297+0000 7f2e0f985500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.301 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.297+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.301 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.297+0000 7f2e0f985500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:08:31.301 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.297+0000 7f2e0f985500 20 zonegroup default 2023-09-22T13:08:31.301 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.297+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.301 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.297+0000 7f2e0f985500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.301 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.297+0000 7f2e0f985500 10 Cannot find current period zone using local zone 2023-09-22T13:08:31.302 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.297+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.302 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.298+0000 7f2e0f985500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.302 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.298+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.302 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.298+0000 7f2e0f985500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.302 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.298+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.302 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.298+0000 7f2e0f985500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:08:31.302 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.298+0000 7f2e0f985500 20 zone default found 2023-09-22T13:08:31.302 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.298+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.302 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.299+0000 7f2e0f985500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.303 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.299+0000 7f2e0f985500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.303 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.299+0000 7f2e0f985500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.303 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.299+0000 7f2e0f985500 20 started sync module instance, tier type = 2023-09-22T13:08:31.303 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.299+0000 7f2e0f985500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:08:31.307 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.305+0000 7f2dfa666700 0 -- 172.21.15.98:0/3100944112 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f2dec029950 msgr2=0x7f2dec0283d0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:08:31.307 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.305+0000 7f2e0f985500 20 add_watcher() i=0 2023-09-22T13:08:31.307 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.305+0000 7f2e0f985500 20 add_watcher() i=1 2023-09-22T13:08:31.307 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.305+0000 7f2e0f985500 20 add_watcher() i=2 2023-09-22T13:08:31.307 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.306+0000 7f2e0f985500 20 add_watcher() i=3 2023-09-22T13:08:31.307 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.306+0000 7f2e0f985500 20 add_watcher() i=4 2023-09-22T13:08:31.307 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.306+0000 7f2e0f985500 20 add_watcher() i=5 2023-09-22T13:08:31.307 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.306+0000 7f2e0f985500 20 add_watcher() i=6 2023-09-22T13:08:31.308 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.307+0000 7f2e0f985500 20 add_watcher() i=7 2023-09-22T13:08:31.308 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.307+0000 7f2e0f985500 2 all 8 watchers are set, enabling cache 2023-09-22T13:08:31.309 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.308+0000 7f2dcbfe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:08:31.309 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.309+0000 7f2e0f985500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:08:31.310 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.309+0000 7f2e0f985500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:08:31.312 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.311+0000 7f2dfae67700 0 -- 172.21.15.98:0/3394670384 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x55a7276314e0 msgr2=0x55a7276376a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:08:31.314 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.313+0000 7f2e0f985500 5 note: GC not initialized 2023-09-22T13:08:31.316 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.314+0000 7f2d3affd700 20 reqs_thread_entry: start 2023-09-22T13:08:31.316 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.315+0000 7f2d38ff9700 20 reqs_thread_entry: start 2023-09-22T13:08:31.316 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.315+0000 7f2e0f985500 20 init_complete bucket index max shards: 1 2023-09-22T13:08:31.316 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.315+0000 7f2e0f985500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:08:31.316 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.315+0000 7f2d2effd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:09:01 2023 2023-09-22T13:08:31.316 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-22T13:08:31.316 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.315+0000 7f2d2e7fc700 20 reqs_thread_entry: start 2023-09-22T13:08:31.327 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.326+0000 7f2e0f985500 20 remove_watcher() i=0 2023-09-22T13:08:31.327 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.326+0000 7f2e0f985500 2 removed watcher, disabling cache 2023-09-22T13:08:31.327 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.327+0000 7f2e0f985500 20 remove_watcher() i=1 2023-09-22T13:08:31.328 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.328+0000 7f2e0f985500 20 remove_watcher() i=2 2023-09-22T13:08:31.329 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.328+0000 7f2e0f985500 20 remove_watcher() i=3 2023-09-22T13:08:31.330 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.329+0000 7f2e0f985500 20 remove_watcher() i=4 2023-09-22T13:08:31.330 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.330+0000 7f2e0f985500 20 remove_watcher() i=5 2023-09-22T13:08:31.331 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.331+0000 7f2e0f985500 20 remove_watcher() i=6 2023-09-22T13:08:31.332 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.331+0000 7f2e0f985500 20 remove_watcher() i=7 2023-09-22T13:08:31.333 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.332+0000 7f2dcbfe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:08:31.336 INFO:teuthology.orchestra.run.smithi098.stdout:[] 2023-09-22T13:08:31.337 DEBUG:tasks.util.rgw: json result: [] 2023-09-22T13:08:31.338 INFO:tasks.rgw:Configuring storage class type = LUKEWARM 2023-09-22T13:08:31.338 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM'] 2023-09-22T13:08:31.338 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM'] 2023-09-22T13:08:31.338 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zonegroup placement add --rgw-zone default --placement-id default-placement --storage-class LUKEWARM 2023-09-22T13:08:31.423 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:08:31.423 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:08:31.429 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.428+0000 7f747a872500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:08:31.429 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.428+0000 7f747a872500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:08:31.434 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.433+0000 7f73aeffd700 20 reqs_thread_entry: start 2023-09-22T13:08:31.434 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.433+0000 7f747a872500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.435 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.434+0000 7f747a872500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.435 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.434+0000 7f747a872500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.436 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.434+0000 7f747a872500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.436 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.435+0000 7f747a872500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.436 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.435+0000 7f747a872500 20 rados_obj.operate() r=0 bl.length=358 2023-09-22T13:08:31.436 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.435+0000 7f747a872500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.437 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.436+0000 7f747a872500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:08:31.440 INFO:teuthology.orchestra.run.smithi098.stdout:[ 2023-09-22T13:08:31.440 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-22T13:08:31.440 INFO:teuthology.orchestra.run.smithi098.stdout: "key": "default-placement", 2023-09-22T13:08:31.440 INFO:teuthology.orchestra.run.smithi098.stdout: "val": { 2023-09-22T13:08:31.441 INFO:teuthology.orchestra.run.smithi098.stdout: "name": "default-placement", 2023-09-22T13:08:31.441 INFO:teuthology.orchestra.run.smithi098.stdout: "tags": [], 2023-09-22T13:08:31.441 INFO:teuthology.orchestra.run.smithi098.stdout: "storage_classes": [ 2023-09-22T13:08:31.441 INFO:teuthology.orchestra.run.smithi098.stdout: "LUKEWARM", 2023-09-22T13:08:31.441 INFO:teuthology.orchestra.run.smithi098.stdout: "STANDARD" 2023-09-22T13:08:31.441 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-22T13:08:31.441 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.441 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.442 INFO:teuthology.orchestra.run.smithi098.stdout:] 2023-09-22T13:08:31.443 DEBUG:tasks.util.rgw: json result: [{'key': 'default-placement', 'val': {'name': 'default-placement', 'tags': [], 'storage_classes': ['LUKEWARM', 'STANDARD']}}] 2023-09-22T13:08:31.443 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM', '--data-pool', 'default.rgw.buckets.data.lukewarm'] 2023-09-22T13:08:31.443 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'LUKEWARM', '--data-pool', 'default.rgw.buckets.data.lukewarm'] 2023-09-22T13:08:31.443 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zone placement add --rgw-zone default --placement-id default-placement --storage-class LUKEWARM --data-pool default.rgw.buckets.data.lukewarm 2023-09-22T13:08:31.491 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:08:31.491 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:08:31.497 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.496+0000 7fdbfc08a500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:08:31.497 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.496+0000 7fdbfc08a500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:08:31.501 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.500+0000 7fdb2ffff700 20 reqs_thread_entry: start 2023-09-22T13:08:31.501 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.500+0000 7fdbfc08a500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.502 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.501+0000 7fdbfc08a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.502 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.501+0000 7fdbfc08a500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.503 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.501+0000 7fdbfc08a500 20 rados_obj.operate() r=0 bl.length=889 2023-09-22T13:08:31.503 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.501+0000 7fdbfc08a500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.503 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.502+0000 7fdbfc08a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.503 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.502+0000 7fdbfc08a500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.504 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.502+0000 7fdbfc08a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.504 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.502+0000 7fdbfc08a500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.505 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.503+0000 7fdbfc08a500 20 rados_obj.operate() r=0 bl.length=382 2023-09-22T13:08:31.507 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-22T13:08:31.508 INFO:teuthology.orchestra.run.smithi098.stdout: "id": "8b3505f7-cd1d-403d-9eb6-cc0558e0de01", 2023-09-22T13:08:31.508 INFO:teuthology.orchestra.run.smithi098.stdout: "name": "default", 2023-09-22T13:08:31.508 INFO:teuthology.orchestra.run.smithi098.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:08:31.508 INFO:teuthology.orchestra.run.smithi098.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:08:31.508 INFO:teuthology.orchestra.run.smithi098.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:08:31.508 INFO:teuthology.orchestra.run.smithi098.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:08:31.508 INFO:teuthology.orchestra.run.smithi098.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:08:31.509 INFO:teuthology.orchestra.run.smithi098.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:08:31.509 INFO:teuthology.orchestra.run.smithi098.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:08:31.509 INFO:teuthology.orchestra.run.smithi098.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:08:31.509 INFO:teuthology.orchestra.run.smithi098.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:08:31.509 INFO:teuthology.orchestra.run.smithi098.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:08:31.509 INFO:teuthology.orchestra.run.smithi098.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:08:31.509 INFO:teuthology.orchestra.run.smithi098.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:08:31.509 INFO:teuthology.orchestra.run.smithi098.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:08:31.509 INFO:teuthology.orchestra.run.smithi098.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:08:31.510 INFO:teuthology.orchestra.run.smithi098.stdout: "system_key": { 2023-09-22T13:08:31.510 INFO:teuthology.orchestra.run.smithi098.stdout: "access_key": "", 2023-09-22T13:08:31.510 INFO:teuthology.orchestra.run.smithi098.stdout: "secret_key": "" 2023-09-22T13:08:31.510 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:08:31.510 INFO:teuthology.orchestra.run.smithi098.stdout: "placement_pools": [ 2023-09-22T13:08:31.510 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-22T13:08:31.510 INFO:teuthology.orchestra.run.smithi098.stdout: "key": "default-placement", 2023-09-22T13:08:31.510 INFO:teuthology.orchestra.run.smithi098.stdout: "val": { 2023-09-22T13:08:31.511 INFO:teuthology.orchestra.run.smithi098.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:08:31.511 INFO:teuthology.orchestra.run.smithi098.stdout: "storage_classes": { 2023-09-22T13:08:31.511 INFO:teuthology.orchestra.run.smithi098.stdout: "LUKEWARM": { 2023-09-22T13:08:31.511 INFO:teuthology.orchestra.run.smithi098.stdout: "data_pool": "default.rgw.buckets.data.lukewarm" 2023-09-22T13:08:31.511 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:08:31.511 INFO:teuthology.orchestra.run.smithi098.stdout: "STANDARD": { 2023-09-22T13:08:31.511 INFO:teuthology.orchestra.run.smithi098.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:08:31.511 INFO:teuthology.orchestra.run.smithi098.stdout: "compression_type": "random" 2023-09-22T13:08:31.511 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.512 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:08:31.512 INFO:teuthology.orchestra.run.smithi098.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:08:31.512 INFO:teuthology.orchestra.run.smithi098.stdout: "index_type": 0 2023-09-22T13:08:31.512 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.512 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.512 INFO:teuthology.orchestra.run.smithi098.stdout: ], 2023-09-22T13:08:31.512 INFO:teuthology.orchestra.run.smithi098.stdout: "realm_id": "", 2023-09-22T13:08:31.512 INFO:teuthology.orchestra.run.smithi098.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:08:31.512 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-22T13:08:31.514 DEBUG:tasks.util.rgw: json result: {'id': '8b3505f7-cd1d-403d-9eb6-cc0558e0de01', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'LUKEWARM': {'data_pool': 'default.rgw.buckets.data.lukewarm'}, 'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:08:31.514 INFO:tasks.rgw:Configuring storage class type = FROZEN 2023-09-22T13:08:31.514 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN'] 2023-09-22T13:08:31.514 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zonegroup', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN'] 2023-09-22T13:08:31.514 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zonegroup placement add --rgw-zone default --placement-id default-placement --storage-class FROZEN 2023-09-22T13:08:31.560 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:08:31.560 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:08:31.566 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.565+0000 7f51ec811500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:08:31.566 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.565+0000 7f51ec811500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:08:31.570 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.569+0000 7f51417fa700 20 reqs_thread_entry: start 2023-09-22T13:08:31.570 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.569+0000 7f51ec811500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.571 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.570+0000 7f51ec811500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.571 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.570+0000 7f51ec811500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.571 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.570+0000 7f51ec811500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.572 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.570+0000 7f51ec811500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.572 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.571+0000 7f51ec811500 20 rados_obj.operate() r=0 bl.length=382 2023-09-22T13:08:31.573 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.571+0000 7f51ec811500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.573 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.572+0000 7f51ec811500 20 rados_obj.operate() r=0 bl.length=956 2023-09-22T13:08:31.576 INFO:teuthology.orchestra.run.smithi098.stdout:[ 2023-09-22T13:08:31.576 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-22T13:08:31.576 INFO:teuthology.orchestra.run.smithi098.stdout: "key": "default-placement", 2023-09-22T13:08:31.577 INFO:teuthology.orchestra.run.smithi098.stdout: "val": { 2023-09-22T13:08:31.577 INFO:teuthology.orchestra.run.smithi098.stdout: "name": "default-placement", 2023-09-22T13:08:31.577 INFO:teuthology.orchestra.run.smithi098.stdout: "tags": [], 2023-09-22T13:08:31.577 INFO:teuthology.orchestra.run.smithi098.stdout: "storage_classes": [ 2023-09-22T13:08:31.577 INFO:teuthology.orchestra.run.smithi098.stdout: "FROZEN", 2023-09-22T13:08:31.577 INFO:teuthology.orchestra.run.smithi098.stdout: "LUKEWARM", 2023-09-22T13:08:31.577 INFO:teuthology.orchestra.run.smithi098.stdout: "STANDARD" 2023-09-22T13:08:31.578 INFO:teuthology.orchestra.run.smithi098.stdout: ] 2023-09-22T13:08:31.578 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.578 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.578 INFO:teuthology.orchestra.run.smithi098.stdout:] 2023-09-22T13:08:31.579 DEBUG:tasks.util.rgw: json result: [{'key': 'default-placement', 'val': {'name': 'default-placement', 'tags': [], 'storage_classes': ['FROZEN', 'LUKEWARM', 'STANDARD']}}] 2023-09-22T13:08:31.579 INFO:tasks.util.rgw:rgwadmin: client.0 : ['zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN', '--data-pool', 'default.rgw.buckets.data.frozen'] 2023-09-22T13:08:31.579 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'ceph', 'zone', 'placement', 'add', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--storage-class', 'FROZEN', '--data-pool', 'default.rgw.buckets.data.frozen'] 2023-09-22T13:08:31.580 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster ceph zone placement add --rgw-zone default --placement-id default-placement --storage-class FROZEN --data-pool default.rgw.buckets.data.frozen 2023-09-22T13:08:31.627 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:08:31.627 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:08:31.633 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.632+0000 7f77eb58f500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:08:31.633 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.632+0000 7f77eb58f500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:08:31.638 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.637+0000 7f771ffff700 20 reqs_thread_entry: start 2023-09-22T13:08:31.638 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.637+0000 7f77eb58f500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.638 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.638+0000 7f77eb58f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.638 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.638+0000 7f77eb58f500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.639 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.638+0000 7f77eb58f500 20 rados_obj.operate() r=0 bl.length=956 2023-09-22T13:08:31.639 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.638+0000 7f77eb58f500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.642 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.639+0000 7f77eb58f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:08:31.642 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.639+0000 7f77eb58f500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.642 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.639+0000 7f77eb58f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:08:31.642 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.639+0000 7f77eb58f500 20 rados->read ofs=0 len=0 2023-09-22T13:08:31.642 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:08:31.640+0000 7f77eb58f500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:08:31.645 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-22T13:08:31.645 INFO:teuthology.orchestra.run.smithi098.stdout: "id": "8b3505f7-cd1d-403d-9eb6-cc0558e0de01", 2023-09-22T13:08:31.645 INFO:teuthology.orchestra.run.smithi098.stdout: "name": "default", 2023-09-22T13:08:31.645 INFO:teuthology.orchestra.run.smithi098.stdout: "domain_root": "default.rgw.meta:root", 2023-09-22T13:08:31.645 INFO:teuthology.orchestra.run.smithi098.stdout: "control_pool": "default.rgw.control", 2023-09-22T13:08:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: "gc_pool": "default.rgw.log:gc", 2023-09-22T13:08:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: "lc_pool": "default.rgw.log:lc", 2023-09-22T13:08:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: "log_pool": "default.rgw.log", 2023-09-22T13:08:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: "intent_log_pool": "default.rgw.log:intent", 2023-09-22T13:08:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: "usage_log_pool": "default.rgw.log:usage", 2023-09-22T13:08:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: "roles_pool": "default.rgw.meta:roles", 2023-09-22T13:08:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: "reshard_pool": "default.rgw.log:reshard", 2023-09-22T13:08:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2023-09-22T13:08:31.646 INFO:teuthology.orchestra.run.smithi098.stdout: "user_email_pool": "default.rgw.meta:users.email", 2023-09-22T13:08:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2023-09-22T13:08:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2023-09-22T13:08:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: "otp_pool": "default.rgw.otp", 2023-09-22T13:08:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: "system_key": { 2023-09-22T13:08:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: "access_key": "", 2023-09-22T13:08:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: "secret_key": "" 2023-09-22T13:08:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:08:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: "placement_pools": [ 2023-09-22T13:08:31.647 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-22T13:08:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: "key": "default-placement", 2023-09-22T13:08:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: "val": { 2023-09-22T13:08:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: "index_pool": "default.rgw.buckets.index", 2023-09-22T13:08:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: "storage_classes": { 2023-09-22T13:08:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: "FROZEN": { 2023-09-22T13:08:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: "data_pool": "default.rgw.buckets.data.frozen" 2023-09-22T13:08:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:08:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: "LUKEWARM": { 2023-09-22T13:08:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: "data_pool": "default.rgw.buckets.data.lukewarm" 2023-09-22T13:08:31.648 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:08:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: "STANDARD": { 2023-09-22T13:08:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: "data_pool": "default.rgw.buckets.data", 2023-09-22T13:08:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: "compression_type": "random" 2023-09-22T13:08:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:08:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2023-09-22T13:08:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: "index_type": 0 2023-09-22T13:08:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.649 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:08:31.650 INFO:teuthology.orchestra.run.smithi098.stdout: ], 2023-09-22T13:08:31.650 INFO:teuthology.orchestra.run.smithi098.stdout: "realm_id": "", 2023-09-22T13:08:31.650 INFO:teuthology.orchestra.run.smithi098.stdout: "notif_pool": "default.rgw.log:notif" 2023-09-22T13:08:31.650 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-22T13:08:31.651 DEBUG:tasks.util.rgw: json result: {'id': '8b3505f7-cd1d-403d-9eb6-cc0558e0de01', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'FROZEN': {'data_pool': 'default.rgw.buckets.data.frozen'}, 'LUKEWARM': {'data_pool': 'default.rgw.buckets.data.lukewarm'}, 'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2023-09-22T13:08:31.651 INFO:tasks.rgw:Starting rgw... 2023-09-22T13:08:31.651 INFO:tasks.rgw:rgw client.0 config is {} 2023-09-22T13:08:31.651 INFO:tasks.rgw:Using beast as radosgw frontend 2023-09-22T13:08:31.652 INFO:tasks.rgw.client.0:Restarting daemon 2023-09-22T13:08:31.652 DEBUG:teuthology.orchestra.run.smithi098:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=80' -n client.0 --cluster ceph -k /etc/ceph/ceph.client.0.keyring --log-file /var/log/ceph/rgw.ceph.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.ceph.client.0.sock --foreground | sudo tee /var/log/ceph/rgw.ceph.client.0.stdout 2>&1 2023-09-22T13:08:31.654 INFO:tasks.rgw.client.0:Started 2023-09-22T13:08:31.655 INFO:tasks.rgw:Polling client.0 until it starts accepting connections on http://smithi098.front.sepia.ceph.com:80/ 2023-09-22T13:08:31.655 DEBUG:teuthology.orchestra.run.smithi098:> curl http://smithi098.front.sepia.ceph.com:80/ 2023-09-22T13:08:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 7 2023-09-22T13:08:31.749 INFO:teuthology.orchestra.run.smithi098.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2023-09-22T13:08:31.750 INFO:teuthology.orchestra.run.smithi098.stderr: Dload Upload Total Spent Left Speed 2023-09-22T13:08:31.750 INFO:teuthology.orchestra.run.smithi098.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed to connect to smithi098.front.sepia.ceph.com port 80: Connection refused 2023-09-22T13:08:32.750 DEBUG:teuthology.orchestra.run.smithi098:> curl http://smithi098.front.sepia.ceph.com:80/ 2023-09-22T13:08:32.774 INFO:teuthology.orchestra.run.smithi098.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2023-09-22T13:08:32.774 INFO:teuthology.orchestra.run.smithi098.stderr: Dload Upload Total Spent Left Speed 2023-09-22T13:08:32.776 INFO:teuthology.orchestra.run.smithi098.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 104k 0 --:--:-- --:--:-- --:--:-- 104k 2023-09-22T13:08:32.777 INFO:teuthology.orchestra.run.smithi098.stdout:anonymous 2023-09-22T13:08:32.777 INFO:teuthology.run_tasks:Running task workunit... 2023-09-22T13:08:32.790 INFO:tasks.workunit:Pulling workunits from ref 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:08:32.790 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-22T13:08:32.790 DEBUG:teuthology.orchestra.run.smithi098:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:08:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T13:08:32.836 INFO:teuthology.orchestra.run.smithi098.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-22T13:08:32.837 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:08:32.895 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-22T13:08:32.896 DEBUG:teuthology.orchestra.run.smithi098:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-22T13:08:32.954 INFO:tasks.workunit:timeout=3h 2023-09-22T13:08:32.954 INFO:tasks.workunit:cleanup=True 2023-09-22T13:08:32.954 DEBUG:teuthology.orchestra.run.smithi098:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:08:33.012 INFO:tasks.workunit.client.0.smithi098.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-22T13:09:32.149 INFO:tasks.workunit.client.0.smithi098.stderr:Note: switching to '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'. 2023-09-22T13:09:32.149 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:32.149 INFO:tasks.workunit.client.0.smithi098.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2023-09-22T13:09:32.149 INFO:tasks.workunit.client.0.smithi098.stderr:changes and commit them, and you can discard any commits you make in this 2023-09-22T13:09:32.149 INFO:tasks.workunit.client.0.smithi098.stderr:state without impacting any branches by switching back to a branch. 2023-09-22T13:09:32.150 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:32.150 INFO:tasks.workunit.client.0.smithi098.stderr:If you want to create a new branch to retain commits you create, you may 2023-09-22T13:09:32.150 INFO:tasks.workunit.client.0.smithi098.stderr:do so (now or later) by using -c with the switch command. Example: 2023-09-22T13:09:32.150 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:32.150 INFO:tasks.workunit.client.0.smithi098.stderr: git switch -c 2023-09-22T13:09:32.150 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:32.150 INFO:tasks.workunit.client.0.smithi098.stderr:Or undo this operation with: 2023-09-22T13:09:32.151 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:32.151 INFO:tasks.workunit.client.0.smithi098.stderr: git switch - 2023-09-22T13:09:32.151 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:32.151 INFO:tasks.workunit.client.0.smithi098.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2023-09-22T13:09:32.151 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:32.151 INFO:tasks.workunit.client.0.smithi098.stderr:HEAD is now at 31c126b7c1c rgw: radosgw-admin bucket check should only print index entries with --check-objects flag 2023-09-22T13:09:32.155 DEBUG:teuthology.orchestra.run.smithi098:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-22T13:09:32.236 INFO:tasks.workunit.client.0.smithi098.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-22T13:09:32.238 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:09:32.238 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-22T13:09:32.846 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-22T13:09:32.912 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-22T13:09:32.950 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:09:32.952 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:09:32.952 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-22T13:09:32.988 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:09:32.992 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:09:32.992 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-22T13:09:33.050 INFO:tasks.workunit:Running workunits matching rgw/run-bucket-check.sh on client.0... 2023-09-22T13:09:33.051 INFO:tasks.workunit:Running workunit rgw/run-bucket-check.sh... 2023-09-22T13:09:33.052 DEBUG:teuthology.orchestra.run.smithi098:workunit test rgw/run-bucket-check.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/run-bucket-check.sh 2023-09-22T13:09:33.112 INFO:tasks.workunit.client.0.smithi098.stderr:++ dirname /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/run-bucket-check.sh 2023-09-22T13:09:33.113 INFO:tasks.workunit.client.0.smithi098.stderr:+ mydir=/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw 2023-09-22T13:09:33.113 INFO:tasks.workunit.client.0.smithi098.stderr:+ python3 -m venv /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw 2023-09-22T13:09:34.614 INFO:tasks.workunit.client.0.smithi098.stderr:+ source /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/bin/activate 2023-09-22T13:09:34.614 INFO:tasks.workunit.client.0.smithi098.stderr:++ deactivate nondestructive 2023-09-22T13:09:34.614 INFO:tasks.workunit.client.0.smithi098.stderr:++ '[' -n '' ']' 2023-09-22T13:09:34.614 INFO:tasks.workunit.client.0.smithi098.stderr:++ '[' -n '' ']' 2023-09-22T13:09:34.615 INFO:tasks.workunit.client.0.smithi098.stderr:++ '[' -n /usr/bin/bash -o -n '' ']' 2023-09-22T13:09:34.615 INFO:tasks.workunit.client.0.smithi098.stderr:++ hash -r 2023-09-22T13:09:34.615 INFO:tasks.workunit.client.0.smithi098.stderr:++ '[' -n '' ']' 2023-09-22T13:09:34.615 INFO:tasks.workunit.client.0.smithi098.stderr:++ unset VIRTUAL_ENV 2023-09-22T13:09:34.615 INFO:tasks.workunit.client.0.smithi098.stderr:++ '[' '!' nondestructive = nondestructive ']' 2023-09-22T13:09:34.615 INFO:tasks.workunit.client.0.smithi098.stderr:++ VIRTUAL_ENV=/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw 2023-09-22T13:09:34.615 INFO:tasks.workunit.client.0.smithi098.stderr:++ export VIRTUAL_ENV 2023-09-22T13:09:34.615 INFO:tasks.workunit.client.0.smithi098.stderr:++ _OLD_VIRTUAL_PATH=/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin 2023-09-22T13:09:34.616 INFO:tasks.workunit.client.0.smithi098.stderr:++ PATH=/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/bin:/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin 2023-09-22T13:09:34.616 INFO:tasks.workunit.client.0.smithi098.stderr:++ export PATH 2023-09-22T13:09:34.616 INFO:tasks.workunit.client.0.smithi098.stderr:++ '[' -n '' ']' 2023-09-22T13:09:34.616 INFO:tasks.workunit.client.0.smithi098.stderr:++ '[' -z '' ']' 2023-09-22T13:09:34.616 INFO:tasks.workunit.client.0.smithi098.stderr:++ _OLD_VIRTUAL_PS1= 2023-09-22T13:09:34.616 INFO:tasks.workunit.client.0.smithi098.stderr:++ '[' 'x(rgw) ' '!=' x ']' 2023-09-22T13:09:34.616 INFO:tasks.workunit.client.0.smithi098.stderr:++ PS1='(rgw) ' 2023-09-22T13:09:34.616 INFO:tasks.workunit.client.0.smithi098.stderr:++ export PS1 2023-09-22T13:09:34.617 INFO:tasks.workunit.client.0.smithi098.stderr:++ '[' -n /usr/bin/bash -o -n '' ']' 2023-09-22T13:09:34.617 INFO:tasks.workunit.client.0.smithi098.stderr:++ hash -r 2023-09-22T13:09:34.617 INFO:tasks.workunit.client.0.smithi098.stderr:+ pip install pip --upgrade 2023-09-22T13:09:35.048 INFO:tasks.workunit.client.0.smithi098.stdout:Collecting pip 2023-09-22T13:09:35.198 INFO:tasks.workunit.client.0.smithi098.stdout: Downloading https://files.pythonhosted.org/packages/a4/6d/6463d49a933f547439d6b5b98b46af8742cc03ae83543e4d7688c2420f8b/pip-21.3.1-py3-none-any.whl (1.7MB) 2023-09-22T13:09:35.489 INFO:tasks.workunit.client.0.smithi098.stdout:Installing collected packages: pip 2023-09-22T13:09:35.489 INFO:tasks.workunit.client.0.smithi098.stdout: Found existing installation: pip 9.0.3 2023-09-22T13:09:35.647 INFO:tasks.workunit.client.0.smithi098.stdout: Uninstalling pip-9.0.3: 2023-09-22T13:09:35.683 INFO:tasks.workunit.client.0.smithi098.stdout: Successfully uninstalled pip-9.0.3 2023-09-22T13:09:36.330 INFO:tasks.workunit.client.0.smithi098.stdout:Successfully installed pip-21.3.1 2023-09-22T13:09:36.612 INFO:tasks.workunit.client.0.smithi098.stderr:You are using pip version 21.3.1, however version 23.2.1 is available. 2023-09-22T13:09:36.613 INFO:tasks.workunit.client.0.smithi098.stderr:You should consider upgrading via the 'pip install --upgrade pip' command. 2023-09-22T13:09:36.643 INFO:tasks.workunit.client.0.smithi098.stderr:+ pip install boto3 2023-09-22T13:09:37.997 INFO:tasks.workunit.client.0.smithi098.stdout:Collecting boto3 2023-09-22T13:09:38.094 INFO:tasks.workunit.client.0.smithi098.stdout: Downloading boto3-1.23.10-py3-none-any.whl (132 kB) 2023-09-22T13:09:39.338 INFO:tasks.workunit.client.0.smithi098.stdout:Collecting botocore<1.27.0,>=1.26.10 2023-09-22T13:09:39.358 INFO:tasks.workunit.client.0.smithi098.stdout: Downloading botocore-1.26.10-py3-none-any.whl (8.8 MB) 2023-09-22T13:09:40.358 INFO:tasks.workunit.client.0.smithi098.stdout:Collecting jmespath<2.0.0,>=0.7.1 2023-09-22T13:09:40.382 INFO:tasks.workunit.client.0.smithi098.stdout: Downloading jmespath-0.10.0-py2.py3-none-any.whl (24 kB) 2023-09-22T13:09:40.490 INFO:tasks.workunit.client.0.smithi098.stdout:Collecting s3transfer<0.6.0,>=0.5.0 2023-09-22T13:09:40.528 INFO:tasks.workunit.client.0.smithi098.stdout: Downloading s3transfer-0.5.2-py3-none-any.whl (79 kB) 2023-09-22T13:09:40.682 INFO:tasks.workunit.client.0.smithi098.stdout:Collecting urllib3<1.27,>=1.25.4 2023-09-22T13:09:40.701 INFO:tasks.workunit.client.0.smithi098.stdout: Downloading urllib3-1.26.16-py2.py3-none-any.whl (143 kB) 2023-09-22T13:09:40.801 INFO:tasks.workunit.client.0.smithi098.stdout:Collecting python-dateutil<3.0.0,>=2.1 2023-09-22T13:09:40.820 INFO:tasks.workunit.client.0.smithi098.stdout: Downloading python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB) 2023-09-22T13:09:41.165 INFO:tasks.workunit.client.0.smithi098.stdout:Collecting six>=1.5 2023-09-22T13:09:41.184 INFO:tasks.workunit.client.0.smithi098.stdout: Downloading six-1.16.0-py2.py3-none-any.whl (11 kB) 2023-09-22T13:09:41.251 INFO:tasks.workunit.client.0.smithi098.stdout:Installing collected packages: six, urllib3, python-dateutil, jmespath, botocore, s3transfer, boto3 2023-09-22T13:09:42.257 INFO:tasks.workunit.client.0.smithi098.stdout:Successfully installed boto3-1.23.10 botocore-1.26.10 jmespath-0.10.0 python-dateutil-2.8.2 s3transfer-0.5.2 six-1.16.0 urllib3-1.26.16 2023-09-22T13:09:42.601 INFO:tasks.workunit.client.0.smithi098.stderr:+ /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/bin/python3 /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_bucket_check.py 2023-09-22T13:09:42.734 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin user create --uid check-tester --display-name "Check Testing" --access-key OJODXSLNX4LUNHQG99PA --secret 3l6ffld34qaymfomuh832j94738aie2x4p2o8h6n 2023-09-22T13:09:42.762 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:42.762 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:42.770 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.769+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.772 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.771+0000 7feae2a7f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:42.772 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.771+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.773 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.772+0000 7feae2a7f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:42.773 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.772+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.773 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.772+0000 7feae2a7f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:42.773 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.772+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.774 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.773+0000 7feae2a7f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:42.786 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.784+0000 7feae2a7f500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:42.786 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.784+0000 7feae2a7f500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:42.786 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.784+0000 7feae2a7f500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:42.786 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.784+0000 7feae2a7f500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:42.786 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.784+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.786 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.784+0000 7feae2a7f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:42.786 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.784+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.786 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.785+0000 7feae2a7f500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:42.787 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.785+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.787 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.785+0000 7feae2a7f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:42.787 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.785+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.789 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.786+0000 7feae2a7f500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:42.789 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.786+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.789 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.786+0000 7feae2a7f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:42.789 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.786+0000 7feae2a7f500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:42.789 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.786+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.789 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.786+0000 7feae2a7f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:42.790 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.786+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.790 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.786+0000 7feae2a7f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:42.790 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.786+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.790 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.787+0000 7feae2a7f500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:42.790 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.787+0000 7feae2a7f500 20 zonegroup default 2023-09-22T13:09:42.790 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.787+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.790 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.787+0000 7feae2a7f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:42.790 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.787+0000 7feae2a7f500 10 Cannot find current period zone using local zone 2023-09-22T13:09:42.790 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.787+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.790 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.787+0000 7feae2a7f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:42.791 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.787+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.791 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.787+0000 7feae2a7f500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:42.791 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.787+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.791 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.788+0000 7feae2a7f500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:42.791 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.788+0000 7feae2a7f500 20 zone default found 2023-09-22T13:09:42.791 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.788+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.791 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.788+0000 7feae2a7f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:42.791 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.788+0000 7feae2a7f500 20 rados->read ofs=0 len=0 2023-09-22T13:09:42.791 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.788+0000 7feae2a7f500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:42.791 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.788+0000 7feae2a7f500 20 started sync module instance, tier type = 2023-09-22T13:09:42.791 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.788+0000 7feae2a7f500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:42.794 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.794+0000 7feacd760700 0 -- 172.21.15.98:0/452799204 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7feac8027bf0 msgr2=0x7feac804c660 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:42.797 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.796+0000 7feae2a7f500 20 add_watcher() i=0 2023-09-22T13:09:42.797 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.796+0000 7feae2a7f500 20 add_watcher() i=1 2023-09-22T13:09:42.797 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.796+0000 7feae2a7f500 20 add_watcher() i=2 2023-09-22T13:09:42.797 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.796+0000 7feae2a7f500 20 add_watcher() i=3 2023-09-22T13:09:42.797 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.796+0000 7feae2a7f500 20 add_watcher() i=4 2023-09-22T13:09:42.798 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.796+0000 7feae2a7f500 20 add_watcher() i=5 2023-09-22T13:09:42.798 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.796+0000 7feae2a7f500 20 add_watcher() i=6 2023-09-22T13:09:42.798 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.797+0000 7feae2a7f500 20 add_watcher() i=7 2023-09-22T13:09:42.798 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.797+0000 7feae2a7f500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:42.799 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.798+0000 7fea9f7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:42.799 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.798+0000 7feae2a7f500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:42.799 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.798+0000 7feae2a7f500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:42.803 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.802+0000 7feace762700 0 -- 172.21.15.98:0/393099162 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x558f2ccbb7e0 msgr2=0x558f2ccc2af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:42.805 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.804+0000 7feae2a7f500 5 note: GC not initialized 2023-09-22T13:09:42.807 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.805+0000 7fea0e7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:42.807 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.805+0000 7fe9fffff700 20 reqs_thread_entry: start 2023-09-22T13:09:42.807 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.805+0000 7feae2a7f500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:42.807 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.805+0000 7feae2a7f500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:42.807 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.805+0000 7fe9fe7fc700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:12 2023 2023-09-22T13:09:42.807 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:42.808 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.805+0000 7fe9fdffb700 20 reqs_thread_entry: start 2023-09-22T13:09:42.808 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.805+0000 7feae2a7f500 20 get_system_obj_state: rctx=0x7ffca46d1288 obj=default.rgw.meta:users.uid:check-tester state=0x558f2cc53f90 s->prefetch_data=0 2023-09-22T13:09:42.808 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.805+0000 7feae2a7f500 10 cache get: name=default.rgw.meta+users.uid+check-tester : miss 2023-09-22T13:09:42.808 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.806+0000 7feae2a7f500 10 cache put: name=default.rgw.meta+users.uid+check-tester info.flags=0x0 2023-09-22T13:09:42.808 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.806+0000 7feae2a7f500 10 adding default.rgw.meta+users.uid+check-tester to cache LRU end 2023-09-22T13:09:42.808 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.806+0000 7feae2a7f500 20 get_system_obj_state: rctx=0x7ffca46d1298 obj=default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA state=0x558f2cc53f90 s->prefetch_data=0 2023-09-22T13:09:42.808 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.806+0000 7feae2a7f500 10 cache get: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA : miss 2023-09-22T13:09:42.808 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.806+0000 7feae2a7f500 10 cache put: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA info.flags=0x0 2023-09-22T13:09:42.808 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.806+0000 7feae2a7f500 10 adding default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA to cache LRU end 2023-09-22T13:09:42.808 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.806+0000 7feae2a7f500 20 get_system_obj_state: rctx=0x7ffca46d0b98 obj=default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA state=0x558f2cc53f90 s->prefetch_data=0 2023-09-22T13:09:42.809 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.806+0000 7feae2a7f500 10 cache get: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA : hit (negative entry) 2023-09-22T13:09:42.809 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.806+0000 7feae2a7f500 20 get_system_obj_state: rctx=0x7ffca46d0ed8 obj=default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA state=0x558f2cc53f90 s->prefetch_data=0 2023-09-22T13:09:42.809 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.806+0000 7feae2a7f500 10 cache get: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA : hit (negative entry) 2023-09-22T13:09:42.810 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.807+0000 7feae2a7f500 10 cache put: name=default.rgw.meta+users.uid+check-tester info.flags=0x17 2023-09-22T13:09:42.810 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.807+0000 7feae2a7f500 10 moving default.rgw.meta+users.uid+check-tester to cache LRU end 2023-09-22T13:09:42.810 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.807+0000 7feae2a7f500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.uid:check-tester, ofs0, ns] 2023-09-22T13:09:42.810 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.807+0000 7feabf7fe700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 154618822656 cookie 94073419986400 notifier 4445 bl.length()=559 2023-09-22T13:09:42.810 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.807+0000 7feabf7fe700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+check-tester info.flags=0x17 2023-09-22T13:09:42.810 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.807+0000 7feabf7fe700 10 rgw watcher librados: moving default.rgw.meta+users.uid+check-tester to cache LRU end 2023-09-22T13:09:42.811 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.809+0000 7feae2a7f500 10 cache put: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA info.flags=0x7 2023-09-22T13:09:42.811 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.809+0000 7feae2a7f500 10 moving default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA to cache LRU end 2023-09-22T13:09:42.811 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.809+0000 7feae2a7f500 10 distributing notification oid=default.rgw.control:notify.0 cni=[op: 0, obj: default.rgw.meta:users.keys:OJODXSLNX4LUNHQG99PA, ofs0, ns] 2023-09-22T13:09:42.811 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.809+0000 7feabeffd700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 154618822656 cookie 94073419967152 notifier 4445 bl.length()=186 2023-09-22T13:09:42.811 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.809+0000 7feabeffd700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA info.flags=0x7 2023-09-22T13:09:42.811 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.809+0000 7feabeffd700 10 rgw watcher librados: moving default.rgw.meta+users.keys+OJODXSLNX4LUNHQG99PA to cache LRU end 2023-09-22T13:09:42.811 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.811+0000 7feae2a7f500 20 remove_watcher() i=0 2023-09-22T13:09:42.812 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.811+0000 7feae2a7f500 2 removed watcher, disabling cache 2023-09-22T13:09:42.812 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.811+0000 7feae2a7f500 20 remove_watcher() i=1 2023-09-22T13:09:42.813 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.812+0000 7feae2a7f500 20 remove_watcher() i=2 2023-09-22T13:09:42.814 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.813+0000 7feae2a7f500 20 remove_watcher() i=3 2023-09-22T13:09:42.815 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.814+0000 7feae2a7f500 20 remove_watcher() i=4 2023-09-22T13:09:42.816 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.815+0000 7feae2a7f500 20 remove_watcher() i=5 2023-09-22T13:09:42.816 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.816+0000 7feae2a7f500 20 remove_watcher() i=6 2023-09-22T13:09:42.817 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.817+0000 7feae2a7f500 20 remove_watcher() i=7 2023-09-22T13:09:42.818 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:42.817+0000 7fea9f7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:43.050 INFO:tasks.workunit.client.0.smithi098.stderr:/home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/lib64/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/ 2023-09-22T13:09:43.050 INFO:tasks.workunit.client.0.smithi098.stderr: warnings.warn(warning, PythonDeprecationWarning) 2023-09-22T13:09:43.050 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: bucket check unlinked does not report normal entries 2023-09-22T13:09:43.050 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:43.051 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --min-age-hours 0 --dump-keys 2023-09-22T13:09:43.077 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:43.077 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:43.085 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.084+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.086 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.086+0000 7fad92758500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:43.087 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.086+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.087 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.086+0000 7fad92758500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:43.087 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.086+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.088 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.087+0000 7fad92758500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:43.088 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.087+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.088 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.087+0000 7fad92758500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:43.099 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.097+0000 7fad92758500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:43.099 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.097+0000 7fad92758500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:43.099 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.097+0000 7fad92758500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:43.099 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.097+0000 7fad92758500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:43.099 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.097+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.100 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.097+0000 7fad92758500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:43.100 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.097+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.100 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.098+0000 7fad92758500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:43.100 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.098+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.100 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.098+0000 7fad92758500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:43.100 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.098+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.100 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.098+0000 7fad92758500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:43.100 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.098+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.102 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.098+0000 7fad92758500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:43.103 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.098+0000 7fad92758500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:43.103 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.098+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.103 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.099+0000 7fad92758500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:43.103 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.099+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.103 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.099+0000 7fad92758500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:43.103 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.099+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.104 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.099+0000 7fad92758500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:43.104 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.099+0000 7fad92758500 20 zonegroup default 2023-09-22T13:09:43.104 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.099+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.104 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.100+0000 7fad92758500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:43.104 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.100+0000 7fad92758500 10 Cannot find current period zone using local zone 2023-09-22T13:09:43.104 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.100+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.104 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.100+0000 7fad92758500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:43.104 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.100+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.105 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.100+0000 7fad92758500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:43.105 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.100+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.105 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.100+0000 7fad92758500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:43.105 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.100+0000 7fad92758500 20 zone default found 2023-09-22T13:09:43.105 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.100+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.105 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.101+0000 7fad92758500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:43.105 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.101+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.105 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.101+0000 7fad92758500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:43.106 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.101+0000 7fad92758500 20 started sync module instance, tier type = 2023-09-22T13:09:43.106 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.101+0000 7fad92758500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:43.109 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.108+0000 7fad92758500 20 add_watcher() i=0 2023-09-22T13:09:43.109 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.108+0000 7fad92758500 20 add_watcher() i=1 2023-09-22T13:09:43.109 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.108+0000 7fad92758500 20 add_watcher() i=2 2023-09-22T13:09:43.109 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.108+0000 7fad92758500 20 add_watcher() i=3 2023-09-22T13:09:43.109 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.108+0000 7fad92758500 20 add_watcher() i=4 2023-09-22T13:09:43.110 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.108+0000 7fad92758500 20 add_watcher() i=5 2023-09-22T13:09:43.110 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.108+0000 7fad92758500 20 add_watcher() i=6 2023-09-22T13:09:43.110 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.108+0000 7fad92758500 20 add_watcher() i=7 2023-09-22T13:09:43.110 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.108+0000 7fad92758500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:43.111 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.110+0000 7fad4f7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:43.111 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.110+0000 7fad92758500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:43.112 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.110+0000 7fad92758500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:43.112 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.111+0000 7fad7e43b700 0 -- >> [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] conn(0x55dfc330f340 msgr2=0x55dfc3311800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0)._try_send injecting socket failure 2023-09-22T13:09:43.117 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.116+0000 7fad92758500 5 note: GC not initialized 2023-09-22T13:09:43.118 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.117+0000 7facbe7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:43.119 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.117+0000 7facaffff700 20 reqs_thread_entry: start 2023-09-22T13:09:43.119 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.117+0000 7fad92758500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:43.119 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.117+0000 7fad92758500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:43.119 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.117+0000 7facae7fc700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:13 2023 2023-09-22T13:09:43.119 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:43.119 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.117+0000 7facadffb700 20 reqs_thread_entry: start 2023-09-22T13:09:43.119 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.117+0000 7fad92758500 20 get_system_obj_state: rctx=0x7ffe3d92f028 obj=default.rgw.meta:root:check-bucket state=0x55dfc327f310 s->prefetch_data=0 2023-09-22T13:09:43.120 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.117+0000 7fad92758500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:09:43.122 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.118+0000 7fad92758500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:09:43.122 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.118+0000 7fad92758500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:43.122 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.118+0000 7fad92758500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:43.122 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.118+0000 7fad92758500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:43.122 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.118+0000 7fad92758500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:09:43.123 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.118+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.123 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:43.123 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:09:43.123 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:43.123 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 20 get_system_obj_state: rctx=0x7ffe3d92f0f8 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55dfc327f310 s->prefetch_data=0 2023-09-22T13:09:43.123 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : miss 2023-09-22T13:09:43.123 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad7dc3a700 0 -- 172.21.15.98:0/2496661217 >> [v2:172.21.15.98:6820/140891,v1:172.21.15.98:6822/140891] conn(0x7fad7801a040 msgr2=0x7fad7801a420 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:43.123 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x16 2023-09-22T13:09:43.124 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:43.124 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:43.124 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:43.124 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:43.124 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:43.124 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:43.124 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:09:43.124 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.119+0000 7fad92758500 20 rados->read ofs=0 len=0 2023-09-22T13:09:43.125 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.120+0000 7fad92758500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:43.125 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.120+0000 7fad92758500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x13 2023-09-22T13:09:43.125 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.120+0000 7fad92758500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:43.125 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.120+0000 7fad92758500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:43.125 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.120+0000 7fad92758500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:43.125 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.120+0000 7fad92758500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 2023-09-22T13:09:43.125 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.120+0000 7fad92758500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:43.125 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.121+0000 7fad92758500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:09:43.126 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.121+0000 7fad92758500 1 NOTICE: finished all shards (0 entries found) 2023-09-22T13:09:43.127 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.122+0000 7fad92758500 20 remove_watcher() i=0 2023-09-22T13:09:43.127 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.122+0000 7fad92758500 2 removed watcher, disabling cache 2023-09-22T13:09:43.127 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.123+0000 7fad92758500 20 remove_watcher() i=1 2023-09-22T13:09:43.127 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.123+0000 7fad92758500 20 remove_watcher() i=2 2023-09-22T13:09:43.127 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.124+0000 7fad92758500 20 remove_watcher() i=3 2023-09-22T13:09:43.127 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.125+0000 7fad92758500 20 remove_watcher() i=4 2023-09-22T13:09:43.128 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.126+0000 7fad92758500 20 remove_watcher() i=5 2023-09-22T13:09:43.128 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.127+0000 7fad92758500 20 remove_watcher() i=6 2023-09-22T13:09:43.129 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.128+0000 7fad92758500 20 remove_watcher() i=7 2023-09-22T13:09:43.129 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:43.129+0000 7fad4f7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:43.135 INFO:tasks.workunit.client.0.smithi098.stderr:ceph config set client rgw_debug_inject_set_olh_err 2 2023-09-22T13:09:43.534 INFO:tasks.workunit.client.0.smithi098.stderr:ceph config set client rgw_debug_inject_olh_cancel_modification_err true 2023-09-22T13:09:44.978 INFO:tasks.workunit.client.0.smithi098.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:09:44.979 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:09:45.004 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:45.005 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:45.012 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.012+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.015 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.014+0000 7f11f29f0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.015 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.014+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.016 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.015+0000 7f11f29f0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.016 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.015+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.016 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.015+0000 7f11f29f0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.016 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.015+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.017 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.016+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.028 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.026+0000 7f11f29f0500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:45.028 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.026+0000 7f11f29f0500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:45.028 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.026+0000 7f11f29f0500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:45.028 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.026+0000 7f11f29f0500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:45.028 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.026+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.026+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.026+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.026+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:45.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.026+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.027+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.027+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.027+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:45.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.027+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.027+0000 7f11f29f0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.027+0000 7f11f29f0500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:45.030 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.027+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.031 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.028+0000 7f11f29f0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.028+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.028+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.028+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.028+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:45.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.028+0000 7f11f29f0500 20 zonegroup default 2023-09-22T13:09:45.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.028+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.028+0000 7f11f29f0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.028+0000 7f11f29f0500 10 Cannot find current period zone using local zone 2023-09-22T13:09:45.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.028+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.029+0000 7f11f29f0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.029+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.029+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.029+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.029+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:45.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.029+0000 7f11f29f0500 20 zone default found 2023-09-22T13:09:45.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.029+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.029+0000 7f11f29f0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.029+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.030+0000 7f11f29f0500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.034 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.030+0000 7f11f29f0500 20 started sync module instance, tier type = 2023-09-22T13:09:45.034 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.030+0000 7f11f29f0500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:45.038 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.037+0000 7f11f29f0500 20 add_watcher() i=0 2023-09-22T13:09:45.038 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.037+0000 7f11f29f0500 20 add_watcher() i=1 2023-09-22T13:09:45.038 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.037+0000 7f11f29f0500 20 add_watcher() i=2 2023-09-22T13:09:45.038 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.037+0000 7f11f29f0500 20 add_watcher() i=3 2023-09-22T13:09:45.038 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.037+0000 7f11f29f0500 20 add_watcher() i=4 2023-09-22T13:09:45.038 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.037+0000 7f11f29f0500 20 add_watcher() i=5 2023-09-22T13:09:45.038 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.037+0000 7f11f29f0500 20 add_watcher() i=6 2023-09-22T13:09:45.039 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.037+0000 7f11f29f0500 20 add_watcher() i=7 2023-09-22T13:09:45.039 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.037+0000 7f11f29f0500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:45.039 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.038+0000 7f11de6d3700 0 -- 172.21.15.98:0/845520905 >> [v2:172.21.15.169:6821/140771,v1:172.21.15.169:6823/140771] conn(0x7f11d4009bd0 msgr2=0x7f11d404c440 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:45.040 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.039+0000 7f11af7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:45.041 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.040+0000 7f11f29f0500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:45.041 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.040+0000 7f11f29f0500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:45.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.044+0000 7f11dded2700 0 -- 172.21.15.98:0/617175310 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x563e7223b9b0 msgr2=0x563e72242480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:45.047 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.046+0000 7f11f29f0500 5 note: GC not initialized 2023-09-22T13:09:45.049 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.047+0000 7f111e7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:45.049 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.047+0000 7f110ffff700 20 reqs_thread_entry: start 2023-09-22T13:09:45.049 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.047+0000 7f11f29f0500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:45.049 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.048+0000 7f11f29f0500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:45.049 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.048+0000 7f110e7fc700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:15 2023 2023-09-22T13:09:45.049 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:45.049 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.048+0000 7f110dffb700 20 reqs_thread_entry: start 2023-09-22T13:09:45.049 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.048+0000 7f11f29f0500 20 get_system_obj_state: rctx=0x7ffff1b42430 obj=default.rgw.meta:root:check-bucket state=0x563e72240da0 s->prefetch_data=0 2023-09-22T13:09:45.051 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.048+0000 7f11f29f0500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.051 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.048+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.051 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.049+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:45.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.049+0000 7f11f29f0500 20 get_system_obj_state: rctx=0x7ffff1b42430 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x563e7212a980 s->prefetch_data=0 2023-09-22T13:09:45.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.049+0000 7f11f29f0500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.049+0000 7f11f29f0500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:45.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.049+0000 7f11f29f0500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:45.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.049+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.049+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:45.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.049+0000 7f11f29f0500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:45.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.049+0000 7f11f29f0500 20 get_system_obj_state: rctx=0x7ffff1b42200 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x563e7212a980 s->prefetch_data=0 2023-09-22T13:09:45.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.050+0000 7f11f29f0500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.050+0000 7f11f29f0500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:45.053 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.050+0000 7f11f29f0500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:45.053 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.050+0000 7f11f29f0500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.053 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.050+0000 7f11f29f0500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:45.053 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.050+0000 7f11f29f0500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:45.053 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.050+0000 7f11f29f0500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:45.054 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.052+0000 7f11f29f0500 20 remove_watcher() i=0 2023-09-22T13:09:45.054 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.052+0000 7f11f29f0500 2 removed watcher, disabling cache 2023-09-22T13:09:45.054 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.053+0000 7f11f29f0500 20 remove_watcher() i=1 2023-09-22T13:09:45.055 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.054+0000 7f11f29f0500 20 remove_watcher() i=2 2023-09-22T13:09:45.055 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.055+0000 7f11f29f0500 20 remove_watcher() i=3 2023-09-22T13:09:45.056 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.056+0000 7f11f29f0500 20 remove_watcher() i=4 2023-09-22T13:09:45.057 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.056+0000 7f11f29f0500 20 remove_watcher() i=5 2023-09-22T13:09:45.058 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.057+0000 7f11f29f0500 20 remove_watcher() i=6 2023-09-22T13:09:45.059 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.058+0000 7f11f29f0500 20 remove_watcher() i=7 2023-09-22T13:09:45.059 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.059+0000 7f11af7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:45.116 INFO:tasks.workunit.client.0.smithi098.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:09:45.117 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:09:45.141 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:45.141 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:45.149 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.148+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.151 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.150+0000 7feeaf0a3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.151 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.150+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.152 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.151+0000 7feeaf0a3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.152 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.151+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.152 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.151+0000 7feeaf0a3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.152 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.151+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.153 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.152+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.163 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.161+0000 7feeaf0a3500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:45.164 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.161+0000 7feeaf0a3500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:45.164 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.161+0000 7feeaf0a3500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:45.164 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.161+0000 7feeaf0a3500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:45.164 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.161+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.164 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.161+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.164 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.161+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.164 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.162+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:45.164 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.162+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.165 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.162+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.165 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.162+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.165 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.162+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:45.165 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.162+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.165 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.163+0000 7feeaf0a3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.165 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.163+0000 7feeaf0a3500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:45.165 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.163+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.167 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.163+0000 7feeaf0a3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.168 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.163+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.168 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.163+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.168 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.163+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.168 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:45.168 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 20 zonegroup default 2023-09-22T13:09:45.168 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.168 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.169 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 10 Cannot find current period zone using local zone 2023-09-22T13:09:45.169 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.169 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.169 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.169 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.169 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.169 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:45.169 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.164+0000 7feeaf0a3500 20 zone default found 2023-09-22T13:09:45.169 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.165+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.170 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.165+0000 7feeaf0a3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.170 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.165+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.170 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.165+0000 7feeaf0a3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.170 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.165+0000 7feeaf0a3500 20 started sync module instance, tier type = 2023-09-22T13:09:45.170 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.165+0000 7feeaf0a3500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:45.172 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.171+0000 7feeaf0a3500 20 add_watcher() i=0 2023-09-22T13:09:45.172 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.171+0000 7feeaf0a3500 20 add_watcher() i=1 2023-09-22T13:09:45.173 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.171+0000 7feeaf0a3500 20 add_watcher() i=2 2023-09-22T13:09:45.173 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.171+0000 7feeaf0a3500 20 add_watcher() i=3 2023-09-22T13:09:45.173 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.171+0000 7feeaf0a3500 20 add_watcher() i=4 2023-09-22T13:09:45.173 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.171+0000 7feeaf0a3500 20 add_watcher() i=5 2023-09-22T13:09:45.173 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.171+0000 7feeaf0a3500 20 add_watcher() i=6 2023-09-22T13:09:45.174 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.172+0000 7fee99d84700 0 -- 172.21.15.98:0/3597623997 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7fee90023040 msgr2=0x7fee90024940 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:45.174 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.172+0000 7feeaf0a3500 20 add_watcher() i=7 2023-09-22T13:09:45.174 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.172+0000 7feeaf0a3500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:45.175 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.174+0000 7fee6bfdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:45.175 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.174+0000 7feeaf0a3500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:45.175 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.174+0000 7feeaf0a3500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:45.179 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.178+0000 7fee99d84700 0 -- 172.21.15.98:0/1479575654 >> [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] conn(0x5630ebfc0b50 msgr2=0x5630ebfb32d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:45.181 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.180+0000 7feeaf0a3500 5 note: GC not initialized 2023-09-22T13:09:45.183 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.181+0000 7feddaffd700 20 reqs_thread_entry: start 2023-09-22T13:09:45.183 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.181+0000 7fedd8ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:45.183 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.181+0000 7feeaf0a3500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:45.183 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.181+0000 7feeaf0a3500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:45.183 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.181+0000 7fedceffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:15 2023 2023-09-22T13:09:45.184 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:45.184 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.181+0000 7fedce7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:45.184 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.181+0000 7feeaf0a3500 20 get_system_obj_state: rctx=0x7ffd62f00d30 obj=default.rgw.meta:root:check-bucket state=0x5630ebf93f40 s->prefetch_data=0 2023-09-22T13:09:45.184 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.182+0000 7feeaf0a3500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.184 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.182+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.184 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.182+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:45.184 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.182+0000 7feeaf0a3500 20 get_system_obj_state: rctx=0x7ffd62f00d30 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x5630ebfb9630 s->prefetch_data=0 2023-09-22T13:09:45.186 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.182+0000 7feeaf0a3500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.186 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.182+0000 7feeaf0a3500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:45.186 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.182+0000 7feeaf0a3500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:45.186 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.182+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.187 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.183+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:45.187 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.183+0000 7feeaf0a3500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:45.187 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.183+0000 7feeaf0a3500 20 get_system_obj_state: rctx=0x7ffd62f00b00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x5630ebfb9630 s->prefetch_data=0 2023-09-22T13:09:45.187 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.183+0000 7feeaf0a3500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.187 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.183+0000 7feeaf0a3500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:45.187 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.183+0000 7feeaf0a3500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:45.187 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.183+0000 7feeaf0a3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.187 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.184+0000 7feeaf0a3500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:45.188 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.184+0000 7feeaf0a3500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:45.188 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.184+0000 7feeaf0a3500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:45.188 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.186+0000 7feeaf0a3500 20 remove_watcher() i=0 2023-09-22T13:09:45.189 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.186+0000 7feeaf0a3500 2 removed watcher, disabling cache 2023-09-22T13:09:45.189 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.187+0000 7feeaf0a3500 20 remove_watcher() i=1 2023-09-22T13:09:45.189 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.188+0000 7feeaf0a3500 20 remove_watcher() i=2 2023-09-22T13:09:45.189 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.188+0000 7feeaf0a3500 20 remove_watcher() i=3 2023-09-22T13:09:45.190 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.189+0000 7feeaf0a3500 20 remove_watcher() i=4 2023-09-22T13:09:45.191 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.190+0000 7feeaf0a3500 20 remove_watcher() i=5 2023-09-22T13:09:45.191 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.191+0000 7feeaf0a3500 20 remove_watcher() i=6 2023-09-22T13:09:45.192 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.191+0000 7feeaf0a3500 20 remove_watcher() i=7 2023-09-22T13:09:45.193 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.192+0000 7fee6bfdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:45.252 INFO:tasks.workunit.client.0.smithi098.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:09:45.253 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object e 2023-09-22T13:09:45.280 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:45.280 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:45.288 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.287+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.290 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.289+0000 7f6ad9741500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.290 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.289+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.291 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.290+0000 7f6ad9741500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.291 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.290+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.291 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.290+0000 7f6ad9741500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.292 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.290+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.292 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.291+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.305 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.301+0000 7f6ad9741500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:45.305 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.301+0000 7f6ad9741500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:45.305 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.301+0000 7f6ad9741500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:45.306 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.301+0000 7f6ad9741500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:45.306 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.301+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.306 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.301+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.306 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.301+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.306 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.301+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:45.306 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.301+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.306 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.302+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.306 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.302+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.307 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.302+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:45.307 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.302+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.307 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.302+0000 7f6ad9741500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.307 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.302+0000 7f6ad9741500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:45.307 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.302+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.307 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.302+0000 7f6ad9741500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.307 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.302+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.307 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.303+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.308 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.303+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.308 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.303+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:45.308 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.303+0000 7f6ad9741500 20 zonegroup default 2023-09-22T13:09:45.308 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.303+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.308 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.303+0000 7f6ad9741500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.308 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.303+0000 7f6ad9741500 10 Cannot find current period zone using local zone 2023-09-22T13:09:45.308 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.303+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.308 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.303+0000 7f6ad9741500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.308 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.303+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.309 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.304+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.309 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.304+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.309 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.304+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:45.309 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.304+0000 7f6ad9741500 20 zone default found 2023-09-22T13:09:45.309 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.304+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.309 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.304+0000 7f6ad9741500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.309 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.304+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.309 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.304+0000 7f6ad9741500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.310 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.304+0000 7f6ad9741500 20 started sync module instance, tier type = 2023-09-22T13:09:45.310 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.304+0000 7f6ad9741500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:45.312 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.311+0000 7f6abffff700 0 -- 172.21.15.98:0/2073596880 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f6ab804c770 msgr2=0x7f6ab804b820 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:45.312 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.311+0000 7f6ad9741500 20 add_watcher() i=0 2023-09-22T13:09:45.312 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.311+0000 7f6ad9741500 20 add_watcher() i=1 2023-09-22T13:09:45.312 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.311+0000 7f6ad9741500 20 add_watcher() i=2 2023-09-22T13:09:45.312 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.311+0000 7f6ad9741500 20 add_watcher() i=3 2023-09-22T13:09:45.312 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.311+0000 7f6ad9741500 20 add_watcher() i=4 2023-09-22T13:09:45.313 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.311+0000 7f6ad9741500 20 add_watcher() i=5 2023-09-22T13:09:45.313 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.311+0000 7f6ad9741500 20 add_watcher() i=6 2023-09-22T13:09:45.313 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.312+0000 7f6ad9741500 20 add_watcher() i=7 2023-09-22T13:09:45.313 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.312+0000 7f6ad9741500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:45.314 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.313+0000 7f6a95fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:45.314 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.314+0000 7f6ad9741500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:45.315 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.314+0000 7f6ad9741500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:45.318 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.317+0000 7f6abffff700 0 -- 172.21.15.98:0/800405086 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x55c27e12aa50 msgr2=0x55c27e11c560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:45.322 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.319+0000 7f6ad9741500 5 note: GC not initialized 2023-09-22T13:09:45.322 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.320+0000 7f6a0cff9700 20 reqs_thread_entry: start 2023-09-22T13:09:45.323 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.320+0000 7f6a027fc700 20 reqs_thread_entry: start 2023-09-22T13:09:45.323 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.321+0000 7f6ad9741500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:45.323 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.321+0000 7f6ad9741500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:45.323 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.321+0000 7f6a00ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:15 2023 2023-09-22T13:09:45.323 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:45.323 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.321+0000 7f69e7fff700 20 reqs_thread_entry: start 2023-09-22T13:09:45.323 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.321+0000 7f6ad9741500 20 get_system_obj_state: rctx=0x7ffcd8161530 obj=default.rgw.meta:root:check-bucket state=0x55c27e125480 s->prefetch_data=0 2023-09-22T13:09:45.324 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.321+0000 7f6ad9741500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.324 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.321+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.324 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.322+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:45.324 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.322+0000 7f6ad9741500 20 get_system_obj_state: rctx=0x7ffcd8161530 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55c27e01d800 s->prefetch_data=0 2023-09-22T13:09:45.325 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.322+0000 7f6ad9741500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.326 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.322+0000 7f6ad9741500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:45.326 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.322+0000 7f6ad9741500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:45.326 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.322+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.326 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.322+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:45.326 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.322+0000 7f6ad9741500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:45.326 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.322+0000 7f6ad9741500 20 get_system_obj_state: rctx=0x7ffcd8161300 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55c27e01d800 s->prefetch_data=0 2023-09-22T13:09:45.326 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.323+0000 7f6ad9741500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.327 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.323+0000 7f6ad9741500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:45.327 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.323+0000 7f6ad9741500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:45.327 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.323+0000 7f6ad9741500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.327 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.323+0000 7f6ad9741500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:45.327 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.323+0000 7f6ad9741500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:45.327 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.323+0000 7f6ad9741500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:45.327 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.325+0000 7f6ad9741500 20 remove_watcher() i=0 2023-09-22T13:09:45.327 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.325+0000 7f6ad9741500 2 removed watcher, disabling cache 2023-09-22T13:09:45.328 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.326+0000 7f6ad9741500 20 remove_watcher() i=1 2023-09-22T13:09:45.328 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.326+0000 7f6ad9741500 20 remove_watcher() i=2 2023-09-22T13:09:45.328 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.327+0000 7f6ad9741500 20 remove_watcher() i=3 2023-09-22T13:09:45.329 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.328+0000 7f6ad9741500 20 remove_watcher() i=4 2023-09-22T13:09:45.329 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.329+0000 7f6ad9741500 20 remove_watcher() i=5 2023-09-22T13:09:45.330 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.329+0000 7f6ad9741500 20 remove_watcher() i=6 2023-09-22T13:09:45.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.330+0000 7f6ad9741500 20 remove_watcher() i=7 2023-09-22T13:09:45.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.331+0000 7f6a95fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:45.388 INFO:tasks.workunit.client.0.smithi098.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:09:45.389 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object f 2023-09-22T13:09:45.414 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:45.414 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:45.422 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.421+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.423 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.422+0000 7ff65734e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.423 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.422+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.424 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.423+0000 7ff65734e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.424 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.423+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.424 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.423+0000 7ff65734e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.425 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.423+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.425 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.424+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.435 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.434+0000 7ff65734e500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:45.435 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.434+0000 7ff65734e500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:45.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.434+0000 7ff65734e500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:45.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.434+0000 7ff65734e500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:45.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.434+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.434+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.434+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.434+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:45.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.434+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.434+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.434+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.440 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.435+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:45.440 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.435+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.440 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.435+0000 7ff65734e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.440 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.435+0000 7ff65734e500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:45.440 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.435+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.440 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.435+0000 7ff65734e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.440 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.435+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.441 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.436+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.441 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.436+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.441 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.436+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:45.441 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.436+0000 7ff65734e500 20 zonegroup default 2023-09-22T13:09:45.441 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.436+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.441 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.436+0000 7ff65734e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.441 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.436+0000 7ff65734e500 10 Cannot find current period zone using local zone 2023-09-22T13:09:45.441 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.436+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.442 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.436+0000 7ff65734e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.442 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.436+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.442 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.437+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:45.442 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.437+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.442 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.437+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:45.442 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.437+0000 7ff65734e500 20 zone default found 2023-09-22T13:09:45.442 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.437+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.442 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.437+0000 7ff65734e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.443 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.437+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.443 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.437+0000 7ff65734e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:45.443 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.437+0000 7ff65734e500 20 started sync module instance, tier type = 2023-09-22T13:09:45.443 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.437+0000 7ff65734e500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:45.445 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.444+0000 7ff64202f700 0 -- 172.21.15.98:0/3191459786 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7ff638014b80 msgr2=0x7ff63804c220 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:45.445 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.444+0000 7ff65734e500 20 add_watcher() i=0 2023-09-22T13:09:45.445 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.444+0000 7ff65734e500 20 add_watcher() i=1 2023-09-22T13:09:45.446 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.444+0000 7ff65734e500 20 add_watcher() i=2 2023-09-22T13:09:45.446 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.444+0000 7ff65734e500 20 add_watcher() i=3 2023-09-22T13:09:45.446 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.444+0000 7ff65734e500 20 add_watcher() i=4 2023-09-22T13:09:45.446 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.444+0000 7ff65734e500 20 add_watcher() i=5 2023-09-22T13:09:45.446 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.444+0000 7ff65734e500 20 add_watcher() i=6 2023-09-22T13:09:45.447 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.445+0000 7ff65734e500 20 add_watcher() i=7 2023-09-22T13:09:45.447 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.445+0000 7ff65734e500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:45.447 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.446+0000 7ff613fe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:45.448 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.447+0000 7ff65734e500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:45.448 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.447+0000 7ff65734e500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:45.451 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.450+0000 7ff642830700 0 -- 172.21.15.98:0/100666782 >> [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] conn(0x565519614380 msgr2=0x56551960b4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:45.453 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.452+0000 7ff65734e500 5 note: GC not initialized 2023-09-22T13:09:45.455 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.453+0000 7ff582ffd700 20 reqs_thread_entry: start 2023-09-22T13:09:45.455 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.453+0000 7ff580ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:45.455 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.453+0000 7ff65734e500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:45.455 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.454+0000 7ff65734e500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:45.455 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.454+0000 7ff576ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:15 2023 2023-09-22T13:09:45.455 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:45.456 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.454+0000 7ff5767fc700 20 reqs_thread_entry: start 2023-09-22T13:09:45.456 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.454+0000 7ff65734e500 20 get_system_obj_state: rctx=0x7ffe59802a70 obj=default.rgw.meta:root:check-bucket state=0x565519466810 s->prefetch_data=0 2023-09-22T13:09:45.457 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.454+0000 7ff65734e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.457 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.454+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.458 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.455+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:45.458 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.455+0000 7ff65734e500 20 get_system_obj_state: rctx=0x7ffe59802a70 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x565519420910 s->prefetch_data=0 2023-09-22T13:09:45.458 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.455+0000 7ff65734e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.458 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.455+0000 7ff65734e500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:45.458 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.455+0000 7ff65734e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:45.458 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.455+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.458 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.456+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:45.459 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.456+0000 7ff65734e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:45.459 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.456+0000 7ff65734e500 20 get_system_obj_state: rctx=0x7ffe59802840 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x565519420910 s->prefetch_data=0 2023-09-22T13:09:45.459 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.456+0000 7ff65734e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:45.459 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.456+0000 7ff65734e500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:45.459 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.456+0000 7ff65734e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:45.459 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.456+0000 7ff65734e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:45.459 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.456+0000 7ff65734e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:45.459 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.456+0000 7ff65734e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:45.459 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.456+0000 7ff65734e500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:45.460 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.458+0000 7ff65734e500 20 remove_watcher() i=0 2023-09-22T13:09:45.461 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.458+0000 7ff65734e500 2 removed watcher, disabling cache 2023-09-22T13:09:45.461 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.459+0000 7ff65734e500 20 remove_watcher() i=1 2023-09-22T13:09:45.461 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.460+0000 7ff65734e500 20 remove_watcher() i=2 2023-09-22T13:09:45.461 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.460+0000 7ff65734e500 20 remove_watcher() i=3 2023-09-22T13:09:45.462 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.461+0000 7ff65734e500 20 remove_watcher() i=4 2023-09-22T13:09:45.463 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.462+0000 7ff65734e500 20 remove_watcher() i=5 2023-09-22T13:09:45.464 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.463+0000 7ff65734e500 20 remove_watcher() i=6 2023-09-22T13:09:45.464 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.464+0000 7ff65734e500 20 remove_watcher() i=7 2023-09-22T13:09:45.465 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:45.464+0000 7ff613fe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:45.475 INFO:tasks.workunit.client.0.smithi098.stderr:ceph config rm client rgw_debug_inject_set_olh_err 2023-09-22T13:09:45.879 INFO:tasks.workunit.client.0.smithi098.stderr:ceph config rm client rgw_debug_inject_olh_cancel_modification_err 2023-09-22T13:09:46.277 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: bucket check unlinked finds unlistable entries 2023-09-22T13:09:46.277 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.278 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --min-age-hours 0 --dump-keys 2023-09-22T13:09:46.304 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:46.305 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:46.313 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.312+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.314 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.314+0000 7f3f8cd24500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.315 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.314+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.315 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.314+0000 7f3f8cd24500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.315 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.314+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.316 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.315+0000 7f3f8cd24500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.316 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.315+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.316 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.315+0000 7f3f8cd24500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.328 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.325+0000 7f3f8cd24500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:46.329 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.325+0000 7f3f8cd24500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:46.329 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.325+0000 7f3f8cd24500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:46.329 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.325+0000 7f3f8cd24500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:46.329 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.325+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.329 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.325+0000 7f3f8cd24500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.329 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.325+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.329 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.326+0000 7f3f8cd24500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.329 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.326+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.330 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.326+0000 7f3f8cd24500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.330 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.326+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.330 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.326+0000 7f3f8cd24500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.330 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.326+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.330 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.326+0000 7f3f8cd24500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.330 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.326+0000 7f3f8cd24500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:46.330 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.326+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.330 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.327+0000 7f3f8cd24500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.330 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.327+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.327+0000 7f3f8cd24500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.327+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.327+0000 7f3f8cd24500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.327+0000 7f3f8cd24500 20 zonegroup default 2023-09-22T13:09:46.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.327+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.327+0000 7f3f8cd24500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.327+0000 7f3f8cd24500 10 Cannot find current period zone using local zone 2023-09-22T13:09:46.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.327+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.328+0000 7f3f8cd24500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.331 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.328+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.332 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.328+0000 7f3f8cd24500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.332 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.328+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.333 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.328+0000 7f3f8cd24500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.333 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.328+0000 7f3f8cd24500 20 zone default found 2023-09-22T13:09:46.333 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.328+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.333 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.328+0000 7f3f8cd24500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.333 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.328+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.334 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.329+0000 7f3f8cd24500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.334 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.329+0000 7f3f8cd24500 20 started sync module instance, tier type = 2023-09-22T13:09:46.334 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.329+0000 7f3f8cd24500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:46.336 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.335+0000 7f3f8cd24500 20 add_watcher() i=0 2023-09-22T13:09:46.337 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.335+0000 7f3f8cd24500 20 add_watcher() i=1 2023-09-22T13:09:46.337 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.335+0000 7f3f8cd24500 20 add_watcher() i=2 2023-09-22T13:09:46.337 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.335+0000 7f3f8cd24500 20 add_watcher() i=3 2023-09-22T13:09:46.337 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.335+0000 7f3f8cd24500 20 add_watcher() i=4 2023-09-22T13:09:46.337 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.335+0000 7f3f8cd24500 20 add_watcher() i=5 2023-09-22T13:09:46.337 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.335+0000 7f3f8cd24500 20 add_watcher() i=6 2023-09-22T13:09:46.338 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.336+0000 7f3f77a05700 0 -- 172.21.15.98:0/142983561 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f3f7002b440 msgr2=0x7f3f7002b880 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:46.338 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.336+0000 7f3f8cd24500 20 add_watcher() i=7 2023-09-22T13:09:46.338 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.336+0000 7f3f8cd24500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:46.338 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.338+0000 7f3f49fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.480 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.338+0000 7f3f8cd24500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:46.480 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.338+0000 7f3f8cd24500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:46.481 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.342+0000 7f3f77a05700 0 -- 172.21.15.98:0/3318520578 >> [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] conn(0x55f5d8551650 msgr2=0x55f5d8543cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:09:46.481 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.344+0000 7f3f8cd24500 5 note: GC not initialized 2023-09-22T13:09:46.481 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.345+0000 7f3ec0ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:46.481 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.345+0000 7f3eb67fc700 20 reqs_thread_entry: start 2023-09-22T13:09:46.481 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.345+0000 7f3f8cd24500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:46.481 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.345+0000 7f3f8cd24500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:46.481 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.345+0000 7f3eb4ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:16 2023 2023-09-22T13:09:46.482 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.482 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.345+0000 7f3e9bfff700 20 reqs_thread_entry: start 2023-09-22T13:09:46.482 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.345+0000 7f3f8cd24500 20 get_system_obj_state: rctx=0x7ffca5018d98 obj=default.rgw.meta:root:check-bucket state=0x55f5d8416b50 s->prefetch_data=0 2023-09-22T13:09:46.482 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.345+0000 7f3f8cd24500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:09:46.482 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:09:46.482 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:46.482 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.482 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.483 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:09:46.483 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.483 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:46.483 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:09:46.483 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:46.483 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 20 get_system_obj_state: rctx=0x7ffca5018e68 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55f5d8416b50 s->prefetch_data=0 2023-09-22T13:09:46.483 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : miss 2023-09-22T13:09:46.484 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x16 2023-09-22T13:09:46.484 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:46.484 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.484 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:46.484 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.484 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:46.484 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:46.484 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:09:46.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.346+0000 7f3f8cd24500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.347+0000 7f3f8cd24500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:46.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.347+0000 7f3f8cd24500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x13 2023-09-22T13:09:46.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.347+0000 7f3f8cd24500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:46.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.347+0000 7f3f8cd24500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.347+0000 7f3f8cd24500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:46.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.347+0000 7f3f8cd24500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 2023-09-22T13:09:46.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.347+0000 7f3f8cd24500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.349+0000 7f3f8cd24500 1 NOTICE: finished shard 0 (4 entries found) 2023-09-22T13:09:46.486 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.349+0000 7f3f8cd24500 1 NOTICE: finished all shards (4 entries found) 2023-09-22T13:09:46.486 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.350+0000 7f3f8cd24500 20 remove_watcher() i=0 2023-09-22T13:09:46.486 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.350+0000 7f3f8cd24500 2 removed watcher, disabling cache 2023-09-22T13:09:46.486 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.351+0000 7f3f8cd24500 20 remove_watcher() i=1 2023-09-22T13:09:46.486 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.352+0000 7f3f8cd24500 20 remove_watcher() i=2 2023-09-22T13:09:46.486 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.352+0000 7f3f8cd24500 20 remove_watcher() i=3 2023-09-22T13:09:46.486 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.353+0000 7f3f8cd24500 20 remove_watcher() i=4 2023-09-22T13:09:46.486 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.354+0000 7f3f8cd24500 20 remove_watcher() i=5 2023-09-22T13:09:46.487 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.355+0000 7f3f8cd24500 20 remove_watcher() i=6 2023-09-22T13:09:46.487 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.356+0000 7f3f8cd24500 20 remove_watcher() i=7 2023-09-22T13:09:46.487 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.356+0000 7f3f49fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.487 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: unlinked entries are not listable 2023-09-22T13:09:46.487 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.487 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: GET returns 404 for unlinked entry keys that have no other versions 2023-09-22T13:09:46.487 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.487 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: bucket check unlinked fixes unlistable entries 2023-09-22T13:09:46.487 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.488 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys 2023-09-22T13:09:46.489 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:46.489 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:46.497 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.496+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.498+0000 7fa04f9f7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.498+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.498+0000 7fa04f9f7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.498+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.499+0000 7fa04f9f7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.499+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.499+0000 7fa04f9f7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.512 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.509+0000 7fa04f9f7500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:46.513 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.509+0000 7fa04f9f7500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:46.513 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.509+0000 7fa04f9f7500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:46.513 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.509+0000 7fa04f9f7500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:46.513 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.509+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.513 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.509+0000 7fa04f9f7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.513 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.509+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.513 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.509+0000 7fa04f9f7500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.513 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.510+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.514 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.510+0000 7fa04f9f7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.514 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.510+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.514 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.510+0000 7fa04f9f7500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.514 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.510+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.514 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.510+0000 7fa04f9f7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.514 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.510+0000 7fa04f9f7500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:46.514 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.510+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.514 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.511+0000 7fa04f9f7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.514 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.511+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.511+0000 7fa04f9f7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.511+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.511+0000 7fa04f9f7500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.511+0000 7fa04f9f7500 20 zonegroup default 2023-09-22T13:09:46.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.511+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.511+0000 7fa04f9f7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.511+0000 7fa04f9f7500 10 Cannot find current period zone using local zone 2023-09-22T13:09:46.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.511+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.512+0000 7fa04f9f7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.512+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.512+0000 7fa04f9f7500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.512+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.512+0000 7fa04f9f7500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.512+0000 7fa04f9f7500 20 zone default found 2023-09-22T13:09:46.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.512+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.513+0000 7fa04f9f7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.520 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.513+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.520 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.513+0000 7fa04f9f7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.520 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.513+0000 7fa04f9f7500 20 started sync module instance, tier type = 2023-09-22T13:09:46.520 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.513+0000 7fa04f9f7500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:46.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.519+0000 7fa04f9f7500 20 add_watcher() i=0 2023-09-22T13:09:46.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.519+0000 7fa04f9f7500 20 add_watcher() i=1 2023-09-22T13:09:46.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.519+0000 7fa04f9f7500 20 add_watcher() i=2 2023-09-22T13:09:46.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.519+0000 7fa04f9f7500 20 add_watcher() i=3 2023-09-22T13:09:46.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.519+0000 7fa04f9f7500 20 add_watcher() i=4 2023-09-22T13:09:46.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.519+0000 7fa04f9f7500 20 add_watcher() i=5 2023-09-22T13:09:46.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.519+0000 7fa04f9f7500 20 add_watcher() i=6 2023-09-22T13:09:46.522 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.520+0000 7fa04f9f7500 20 add_watcher() i=7 2023-09-22T13:09:46.522 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.520+0000 7fa04f9f7500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:46.522 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.520+0000 7fa03b6da700 0 -- 172.21.15.98:0/2437393093 >> [v2:172.21.15.169:6821/140771,v1:172.21.15.169:6823/140771] conn(0x7fa02c01b890 msgr2=0x7fa02c00c340 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:09:46.522 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.522+0000 7fa00bfe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.523 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.522+0000 7fa04f9f7500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:46.523 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.522+0000 7fa04f9f7500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:46.527 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.526+0000 7fa03b6da700 0 -- 172.21.15.98:0/258985786 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x558b8b938610 msgr2=0x558b8b93ffa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:46.529 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.528+0000 7fa04f9f7500 5 note: GC not initialized 2023-09-22T13:09:46.531 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.529+0000 7f9f7affd700 20 reqs_thread_entry: start 2023-09-22T13:09:46.531 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.529+0000 7f9f78ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:46.531 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.529+0000 7fa04f9f7500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:46.531 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.529+0000 7fa04f9f7500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:46.531 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.529+0000 7f9f6effd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:17 2023 2023-09-22T13:09:46.532 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.532 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.529+0000 7f9f6e7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:46.532 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.529+0000 7fa04f9f7500 20 get_system_obj_state: rctx=0x7ffebe5e97d8 obj=default.rgw.meta:root:check-bucket state=0x558b8b8a2f70 s->prefetch_data=0 2023-09-22T13:09:46.532 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.529+0000 7fa04f9f7500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:09:46.532 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:09:46.532 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:46.532 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.532 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.532 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:09:46.532 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.533 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:46.533 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:09:46.533 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:46.533 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 20 get_system_obj_state: rctx=0x7ffebe5e98a8 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x558b8b8a2f70 s->prefetch_data=0 2023-09-22T13:09:46.533 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : miss 2023-09-22T13:09:46.534 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x16 2023-09-22T13:09:46.534 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:46.535 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.535 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:46.535 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.535 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:46.535 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:46.535 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.530+0000 7fa04f9f7500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:09:46.535 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.531+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.535 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.531+0000 7fa04f9f7500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:46.535 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.531+0000 7fa04f9f7500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x13 2023-09-22T13:09:46.535 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.531+0000 7fa04f9f7500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:46.536 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.531+0000 7fa04f9f7500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.536 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.531+0000 7fa04f9f7500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:46.536 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.531+0000 7fa04f9f7500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 2023-09-22T13:09:46.536 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.531+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.536 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.532+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.536 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.532+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:c state=0x558b8b9da058 s->prefetch_data=0 2023-09-22T13:09:46.536 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.533+0000 7fa04f9f7500 20 get_obj_state: setting s->obj_tag to fwz1kdf2exrv8oz808vcd86flttnp9y9 2023-09-22T13:09:46.536 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.533+0000 7fa04f9f7500 20 get_obj_state_impl: setting s->olh_tag to lsnbnq1g01z87mbv2lynjdm52bc7u3pd 2023-09-22T13:09:46.536 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.533+0000 7fa04f9f7500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=lsnbnq1g01z87mbv2lynjdm52bc7u3pd 2023-09-22T13:09:46.537 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.534+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.537 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.535+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.538 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.535+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.538 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.535+0000 7fa04f9f7500 20 olh_log_entry: epoch=3 op=3 key=c[FzGS8FYJBOej4Cp8g04X64n1.8sXYu.] 2023-09-22T13:09:46.538 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.535+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:_:FzGS8FYJBOej4Cp8g04X64n1.8sXYu._c state=0x558b8b9e37f8 s->prefetch_data=0 2023-09-22T13:09:46.538 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.535+0000 7fa04f9f7500 10 manifest: total_size = 22 2023-09-22T13:09:46.538 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.535+0000 7fa04f9f7500 20 get_obj_state: setting s->obj_tag to 8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4430.3628886209210101956 2023-09-22T13:09:46.538 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.535+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:_:FzGS8FYJBOej4Cp8g04X64n1.8sXYu._c state=0x558b8b9e37f8 s->prefetch_data=0 2023-09-22T13:09:46.538 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.535+0000 7fa04f9f7500 20 prepare_atomic_modification: state is not atomic. state=0x558b8b9e37f8 2023-09-22T13:09:46.538 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.536+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.539 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : miss 2023-09-22T13:09:46.539 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.539 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.540 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 10 cache put: name=default.rgw.log++bucket.sync-source-hints.check-bucket info.flags=0x0 2023-09-22T13:09:46.540 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 10 adding default.rgw.log++bucket.sync-source-hints.check-bucket to cache LRU end 2023-09-22T13:09:46.540 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : miss 2023-09-22T13:09:46.540 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.540 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.540 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 10 cache put: name=default.rgw.log++bucket.sync-target-hints.check-bucket info.flags=0x0 2023-09-22T13:09:46.540 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 10 adding default.rgw.log++bucket.sync-target-hints.check-bucket to cache LRU end 2023-09-22T13:09:46.540 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:46.541 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:46.541 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.538+0000 7fa04f9f7500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:46.542 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.539+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.542 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.539+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.542 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.540+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.542 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.540+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.542 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.540+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.543 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.540+0000 7fa04f9f7500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:46.543 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.540+0000 7fa04f9f7500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:46.543 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.540+0000 7fa04f9f7500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:46.543 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.541+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.543 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.541+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:d state=0x558b8b9da058 s->prefetch_data=0 2023-09-22T13:09:46.543 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.541+0000 7fa04f9f7500 20 get_obj_state: setting s->obj_tag to 9qz07ey9ho4ol5rvep5dza03rtzr38wy 2023-09-22T13:09:46.543 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.541+0000 7fa04f9f7500 20 get_obj_state_impl: setting s->olh_tag to 74fro5yhmgfgyk5mjfl5mb630yt3fn9j 2023-09-22T13:09:46.543 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.541+0000 7fa04f9f7500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=74fro5yhmgfgyk5mjfl5mb630yt3fn9j 2023-09-22T13:09:46.546 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.542+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.546 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.543+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.546 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.543+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.546 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.544+0000 7fa04f9f7500 20 olh_log_entry: epoch=3 op=3 key=d[PD.M2oEXO6TZiAkZtzT00IV0RZPVR.d] 2023-09-22T13:09:46.546 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.544+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:_:PD.M2oEXO6TZiAkZtzT00IV0RZPVR.d_d state=0x558b8b9e3be8 s->prefetch_data=0 2023-09-22T13:09:46.546 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.544+0000 7fa04f9f7500 10 manifest: total_size = 9 2023-09-22T13:09:46.546 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.544+0000 7fa04f9f7500 20 get_obj_state: setting s->obj_tag to 8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4430.2444160690510582319 2023-09-22T13:09:46.546 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.544+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:_:PD.M2oEXO6TZiAkZtzT00IV0RZPVR.d_d state=0x558b8b9e3be8 s->prefetch_data=0 2023-09-22T13:09:46.547 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.544+0000 7fa04f9f7500 20 prepare_atomic_modification: state is not atomic. state=0x558b8b9e3be8 2023-09-22T13:09:46.547 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.544+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.547 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.546+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.547 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.546+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.547 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.546+0000 7fa04f9f7500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:46.547 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.546+0000 7fa04f9f7500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:46.548 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.546+0000 7fa04f9f7500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:46.548 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.547+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.548 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.547+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.550 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.548+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.550 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.548+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.550 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.548+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.550 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.548+0000 7fa04f9f7500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:46.550 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.548+0000 7fa04f9f7500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:46.550 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.548+0000 7fa04f9f7500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:46.550 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.549+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.550 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.549+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:e state=0x558b8b9da058 s->prefetch_data=0 2023-09-22T13:09:46.551 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.549+0000 7fa04f9f7500 20 get_obj_state: setting s->obj_tag to 67tc3azgn6wo0wfbvm716mdd3kaguoks 2023-09-22T13:09:46.551 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.549+0000 7fa04f9f7500 20 get_obj_state_impl: setting s->olh_tag to m6r8cvcybyi7ra1oss4mkw4j71ez2wsp 2023-09-22T13:09:46.551 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.549+0000 7fa04f9f7500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=m6r8cvcybyi7ra1oss4mkw4j71ez2wsp 2023-09-22T13:09:46.551 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.550+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.552 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.551+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.552 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.551+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.553 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.552+0000 7fa04f9f7500 20 olh_log_entry: epoch=2 op=2 key=e[] 2023-09-22T13:09:46.553 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.552+0000 7fa04f9f7500 20 olh_log_entry: epoch=2 op=3 key=e[OzG1OHibVC5gMCQ7dTl6grbbOxFHtAV] 2023-09-22T13:09:46.553 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.552+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:_:OzG1OHibVC5gMCQ7dTl6grbbOxFHtAV_e state=0x558b8b9e3b38 s->prefetch_data=0 2023-09-22T13:09:46.553 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.552+0000 7fa04f9f7500 10 manifest: total_size = 9 2023-09-22T13:09:46.553 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.552+0000 7fa04f9f7500 20 get_obj_state: setting s->obj_tag to 8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4430.14208726419802948246 2023-09-22T13:09:46.554 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.552+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:_:OzG1OHibVC5gMCQ7dTl6grbbOxFHtAV_e state=0x558b8b9e3b38 s->prefetch_data=0 2023-09-22T13:09:46.554 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.552+0000 7fa04f9f7500 20 prepare_atomic_modification: state is not atomic. state=0x558b8b9e3b38 2023-09-22T13:09:46.554 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.552+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.554 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.554+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.555 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.554+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.555 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.554+0000 7fa04f9f7500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:46.555 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.554+0000 7fa04f9f7500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:46.555 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.554+0000 7fa04f9f7500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:46.556 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.555+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:e state=0x558b8b9da058 s->prefetch_data=0 2023-09-22T13:09:46.556 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.555+0000 7fa04f9f7500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=m6r8cvcybyi7ra1oss4mkw4j71ez2wsp 2023-09-22T13:09:46.558 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.557+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.559 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.558+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.559 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.558+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.559 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.558+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.560 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.558+0000 7fa04f9f7500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:46.560 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.558+0000 7fa04f9f7500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:46.560 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.558+0000 7fa04f9f7500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:46.560 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.559+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.560 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.559+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:f state=0x558b8b9da058 s->prefetch_data=0 2023-09-22T13:09:46.561 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.559+0000 7fa04f9f7500 20 get_obj_state: setting s->obj_tag to uc2375vq68btwnsrz5qdxnvznebka1ub 2023-09-22T13:09:46.561 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.559+0000 7fa04f9f7500 20 get_obj_state_impl: setting s->olh_tag to cnzhm99cvp0ftzdxom5qpryr3c884con 2023-09-22T13:09:46.561 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.559+0000 7fa04f9f7500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=cnzhm99cvp0ftzdxom5qpryr3c884con 2023-09-22T13:09:46.561 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.560+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.562 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.562+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.563 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.562+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.563 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.562+0000 7fa04f9f7500 20 olh_log_entry: epoch=2 op=2 key=f[] 2023-09-22T13:09:46.564 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.562+0000 7fa04f9f7500 20 olh_log_entry: epoch=2 op=3 key=f[MSf3e9EieLzWQUWBrRA5FO02kzGEeeD] 2023-09-22T13:09:46.564 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.562+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:_:MSf3e9EieLzWQUWBrRA5FO02kzGEeeD_f state=0x558b8b9e3b38 s->prefetch_data=0 2023-09-22T13:09:46.564 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.562+0000 7fa04f9f7500 10 manifest: total_size = 11 2023-09-22T13:09:46.564 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.562+0000 7fa04f9f7500 20 get_obj_state: setting s->obj_tag to 8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4430.9331403043589907012 2023-09-22T13:09:46.564 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.562+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:_:MSf3e9EieLzWQUWBrRA5FO02kzGEeeD_f state=0x558b8b9e3b38 s->prefetch_data=0 2023-09-22T13:09:46.564 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.562+0000 7fa04f9f7500 20 prepare_atomic_modification: state is not atomic. state=0x558b8b9e3b38 2023-09-22T13:09:46.564 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.562+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.565 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.564+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.565 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.564+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.566 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.564+0000 7fa04f9f7500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:46.566 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.564+0000 7fa04f9f7500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:46.566 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.564+0000 7fa04f9f7500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:46.567 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.566+0000 7fa04f9f7500 20 get_obj_state: rctx=0x558b8b9d8e70 obj=check-bucket:f state=0x558b8b9da058 s->prefetch_data=0 2023-09-22T13:09:46.567 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.566+0000 7fa04f9f7500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=cnzhm99cvp0ftzdxom5qpryr3c884con 2023-09-22T13:09:46.569 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.568+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.570 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.569+0000 7fa04f9f7500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.571 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.569+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.571 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.569+0000 7fa04f9f7500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:46.571 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.569+0000 7fa04f9f7500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:46.571 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.569+0000 7fa04f9f7500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:46.571 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.569+0000 7fa04f9f7500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:46.571 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.569+0000 7fa04f9f7500 1 NOTICE: finished shard 0 (4 entries removed) 2023-09-22T13:09:46.571 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.569+0000 7fa04f9f7500 1 NOTICE: finished all shards (4 entries removed) 2023-09-22T13:09:46.572 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.570+0000 7fa04f9f7500 20 remove_watcher() i=0 2023-09-22T13:09:46.572 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.570+0000 7fa04f9f7500 2 removed watcher, disabling cache 2023-09-22T13:09:46.572 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.571+0000 7fa04f9f7500 20 remove_watcher() i=1 2023-09-22T13:09:46.573 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.572+0000 7fa04f9f7500 20 remove_watcher() i=2 2023-09-22T13:09:46.573 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.573+0000 7fa04f9f7500 20 remove_watcher() i=3 2023-09-22T13:09:46.574 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.573+0000 7fa04f9f7500 20 remove_watcher() i=4 2023-09-22T13:09:46.575 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.574+0000 7fa04f9f7500 20 remove_watcher() i=5 2023-09-22T13:09:46.575 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.575+0000 7fa04f9f7500 20 remove_watcher() i=6 2023-09-22T13:09:46.576 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.576+0000 7fa04f9f7500 20 remove_watcher() i=7 2023-09-22T13:09:46.577 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.576+0000 7fa00bfe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.592 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: bucket check unlinked does not affect normal entries 2023-09-22T13:09:46.592 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.648 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: bucket check unlinked does not find new unlistable entries after fix 2023-09-22T13:09:46.648 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.649 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --min-age-hours 0 --dump-keys 2023-09-22T13:09:46.674 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:46.675 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:46.683 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.682+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.684 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.684+0000 7f97beaa3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.684 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.684+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.685 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.684+0000 7f97beaa3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.685 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.684+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.685 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.685+0000 7f97beaa3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.685 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.685+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.686 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.685+0000 7f97beaa3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.695+0000 7f97beaa3500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:46.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.695+0000 7f97beaa3500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:46.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.695+0000 7f97beaa3500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:46.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.695+0000 7f97beaa3500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:46.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.695+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.695+0000 7f97beaa3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.695+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.695+0000 7f97beaa3500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.695+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.696+0000 7f97beaa3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.696+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.696+0000 7f97beaa3500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.696+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.696+0000 7f97beaa3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.696+0000 7f97beaa3500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:46.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.696+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.696+0000 7f97beaa3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.696+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.697+0000 7f97beaa3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.697+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.697+0000 7f97beaa3500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.697+0000 7f97beaa3500 20 zonegroup default 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.697+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.697+0000 7f97beaa3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.697+0000 7f97beaa3500 10 Cannot find current period zone using local zone 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.697+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.697+0000 7f97beaa3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.697+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.698+0000 7f97beaa3500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.698+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.701 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.698+0000 7f97beaa3500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.701 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.698+0000 7f97beaa3500 20 zone default found 2023-09-22T13:09:46.701 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.698+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.701 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.698+0000 7f97beaa3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.701 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.698+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.701 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.698+0000 7f97beaa3500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.701 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.698+0000 7f97beaa3500 20 started sync module instance, tier type = 2023-09-22T13:09:46.702 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.698+0000 7f97beaa3500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:46.706 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.705+0000 7f97a9784700 0 -- 172.21.15.98:0/2294103975 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f979c0073c0 msgr2=0x7f979c007800 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:46.706 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.705+0000 7f97beaa3500 20 add_watcher() i=0 2023-09-22T13:09:46.706 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.705+0000 7f97beaa3500 20 add_watcher() i=1 2023-09-22T13:09:46.706 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.705+0000 7f97beaa3500 20 add_watcher() i=2 2023-09-22T13:09:46.706 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.705+0000 7f97beaa3500 20 add_watcher() i=3 2023-09-22T13:09:46.706 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.705+0000 7f97beaa3500 20 add_watcher() i=4 2023-09-22T13:09:46.706 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.705+0000 7f97beaa3500 20 add_watcher() i=5 2023-09-22T13:09:46.706 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.705+0000 7f97beaa3500 20 add_watcher() i=6 2023-09-22T13:09:46.707 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.706+0000 7f97beaa3500 20 add_watcher() i=7 2023-09-22T13:09:46.707 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.706+0000 7f97beaa3500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:46.708 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.707+0000 7f977b7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.708 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.708+0000 7f97beaa3500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:46.708 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.708+0000 7f97beaa3500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:46.712 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.711+0000 7f97aa786700 0 -- 172.21.15.98:0/868190288 >> [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] conn(0x55a5552ecf10 msgr2=0x55a5553073b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:46.714 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.713+0000 7f97beaa3500 5 note: GC not initialized 2023-09-22T13:09:46.715 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.714+0000 7f96ea7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:46.716 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.714+0000 7f96dbfff700 20 reqs_thread_entry: start 2023-09-22T13:09:46.716 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.714+0000 7f97beaa3500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:46.716 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.715+0000 7f97beaa3500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:46.716 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.715+0000 7f96da7fc700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:16 2023 2023-09-22T13:09:46.716 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.716 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.715+0000 7f96d9ffb700 20 reqs_thread_entry: start 2023-09-22T13:09:46.716 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.715+0000 7f97beaa3500 20 get_system_obj_state: rctx=0x7ffe4165f158 obj=default.rgw.meta:root:check-bucket state=0x55a5552c9350 s->prefetch_data=0 2023-09-22T13:09:46.716 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.715+0000 7f97beaa3500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:09:46.718 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.715+0000 7f97beaa3500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:09:46.718 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.715+0000 7f97beaa3500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:46.718 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.715+0000 7f97beaa3500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.715+0000 7f97beaa3500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.715+0000 7f97beaa3500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:09:46.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.715+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:46.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:09:46.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:46.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 20 get_system_obj_state: rctx=0x7ffe4165f228 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55a5552c9350 s->prefetch_data=0 2023-09-22T13:09:46.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : miss 2023-09-22T13:09:46.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x16 2023-09-22T13:09:46.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:46.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.720 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:46.720 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.720 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:46.720 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:46.720 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:09:46.720 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.716+0000 7f97beaa3500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.720 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.717+0000 7f97beaa3500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:46.720 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.717+0000 7f97beaa3500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x13 2023-09-22T13:09:46.720 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.717+0000 7f97beaa3500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:46.720 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.717+0000 7f97beaa3500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.720 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.717+0000 7f97beaa3500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:46.721 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.717+0000 7f97beaa3500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 2023-09-22T13:09:46.721 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.717+0000 7f97beaa3500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.721 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.717+0000 7f97beaa3500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:09:46.721 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.717+0000 7f97beaa3500 1 NOTICE: finished all shards (0 entries found) 2023-09-22T13:09:46.721 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.718+0000 7f97beaa3500 20 remove_watcher() i=0 2023-09-22T13:09:46.722 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.718+0000 7f97beaa3500 2 removed watcher, disabling cache 2023-09-22T13:09:46.722 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.719+0000 7f97beaa3500 20 remove_watcher() i=1 2023-09-22T13:09:46.722 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.720+0000 7f97beaa3500 20 remove_watcher() i=2 2023-09-22T13:09:46.722 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.721+0000 7f97beaa3500 20 remove_watcher() i=3 2023-09-22T13:09:46.722 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.721+0000 7f97beaa3500 20 remove_watcher() i=4 2023-09-22T13:09:46.723 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.722+0000 7f97beaa3500 20 remove_watcher() i=5 2023-09-22T13:09:46.723 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.723+0000 7f97beaa3500 20 remove_watcher() i=6 2023-09-22T13:09:46.724 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.724+0000 7f97beaa3500 20 remove_watcher() i=7 2023-09-22T13:09:46.725 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.724+0000 7f977b7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.754 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:09:46.778 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:46.778 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:46.786 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.785+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.787 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.787+0000 7f1be2a14500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.787 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.787+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.788 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.787+0000 7f1be2a14500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.788 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.787+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.788 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.787+0000 7f1be2a14500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.788 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.787+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.789 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.788+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.799 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.798+0000 7f1be2a14500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:46.799 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.798+0000 7f1be2a14500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:46.799 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.798+0000 7f1be2a14500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:46.799 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.798+0000 7f1be2a14500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:46.800 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.798+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.800 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.798+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.800 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.798+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.800 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.799+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.800 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.799+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.801 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.799+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.801 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.799+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.801 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.799+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.801 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.799+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.802 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.799+0000 7f1be2a14500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.802 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.799+0000 7f1be2a14500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:46.802 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.799+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.802 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.800+0000 7f1be2a14500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.802 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.800+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.802 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.800+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.802 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.800+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.802 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.800+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.802 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.800+0000 7f1be2a14500 20 zonegroup default 2023-09-22T13:09:46.802 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.800+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.802 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.801+0000 7f1be2a14500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.803 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.801+0000 7f1be2a14500 10 Cannot find current period zone using local zone 2023-09-22T13:09:46.803 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.801+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.804 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.801+0000 7f1be2a14500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.804 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.801+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.804 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.801+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.804 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.801+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.804 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.801+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.804 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.801+0000 7f1be2a14500 20 zone default found 2023-09-22T13:09:46.804 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.801+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.804 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.802+0000 7f1be2a14500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.804 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.802+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.804 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.802+0000 7f1be2a14500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.805 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.802+0000 7f1be2a14500 20 started sync module instance, tier type = 2023-09-22T13:09:46.805 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.802+0000 7f1be2a14500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:46.809 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.808+0000 7f1bcd6f5700 0 -- 172.21.15.98:0/1066778231 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f1bc403a920 msgr2=0x7f1bc402a410 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:46.809 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.808+0000 7f1be2a14500 20 add_watcher() i=0 2023-09-22T13:09:46.809 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.808+0000 7f1be2a14500 20 add_watcher() i=1 2023-09-22T13:09:46.809 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.808+0000 7f1be2a14500 20 add_watcher() i=2 2023-09-22T13:09:46.809 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.808+0000 7f1be2a14500 20 add_watcher() i=3 2023-09-22T13:09:46.809 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.808+0000 7f1be2a14500 20 add_watcher() i=4 2023-09-22T13:09:46.809 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.808+0000 7f1be2a14500 20 add_watcher() i=5 2023-09-22T13:09:46.809 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.808+0000 7f1be2a14500 20 add_watcher() i=6 2023-09-22T13:09:46.810 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.809+0000 7f1be2a14500 20 add_watcher() i=7 2023-09-22T13:09:46.810 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.809+0000 7f1be2a14500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:46.811 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.811+0000 7f1b9f7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.812 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.811+0000 7f1be2a14500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:46.812 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.811+0000 7f1be2a14500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:46.816 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.815+0000 7f1bcdef6700 0 -- 172.21.15.98:0/526123015 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x55a845dd0cd0 msgr2=0x55a845dcc510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:46.817 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.817+0000 7f1be2a14500 5 note: GC not initialized 2023-09-22T13:09:46.819 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.818+0000 7f1b0e7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:46.819 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.818+0000 7f1afffff700 20 reqs_thread_entry: start 2023-09-22T13:09:46.819 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.818+0000 7f1be2a14500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:46.819 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.818+0000 7f1be2a14500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:46.819 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.818+0000 7f1afe7fc700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:16 2023 2023-09-22T13:09:46.820 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.820 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.818+0000 7f1afdffb700 20 reqs_thread_entry: start 2023-09-22T13:09:46.820 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.818+0000 7f1be2a14500 20 get_system_obj_state: rctx=0x7ffd2d1d3ad0 obj=default.rgw.meta:root:check-bucket state=0x55a845da9160 s->prefetch_data=0 2023-09-22T13:09:46.821 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.819+0000 7f1be2a14500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.821 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.819+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.821 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.819+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:46.821 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.819+0000 7f1be2a14500 20 get_system_obj_state: rctx=0x7ffd2d1d3ad0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55a845cc0980 s->prefetch_data=0 2023-09-22T13:09:46.821 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.820+0000 7f1be2a14500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.821 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.820+0000 7f1be2a14500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:46.821 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.820+0000 7f1be2a14500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:46.821 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.820+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.822 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.820+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:46.822 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.820+0000 7f1be2a14500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:46.822 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.820+0000 7f1be2a14500 20 get_system_obj_state: rctx=0x7ffd2d1d38a0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55a845cc0980 s->prefetch_data=0 2023-09-22T13:09:46.822 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.820+0000 7f1be2a14500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.823 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.820+0000 7f1be2a14500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:46.823 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.820+0000 7f1be2a14500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:46.823 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.820+0000 7f1be2a14500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.823 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.821+0000 7f1be2a14500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:46.823 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.821+0000 7f1be2a14500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:46.823 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.821+0000 7f1be2a14500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.823 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.823+0000 7f1be2a14500 20 remove_watcher() i=0 2023-09-22T13:09:46.823 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.823+0000 7f1be2a14500 2 removed watcher, disabling cache 2023-09-22T13:09:46.824 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.823+0000 7f1be2a14500 20 remove_watcher() i=1 2023-09-22T13:09:46.825 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.824+0000 7f1be2a14500 20 remove_watcher() i=2 2023-09-22T13:09:46.825 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.825+0000 7f1be2a14500 20 remove_watcher() i=3 2023-09-22T13:09:46.826 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.826+0000 7f1be2a14500 20 remove_watcher() i=4 2023-09-22T13:09:46.827 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.826+0000 7f1be2a14500 20 remove_watcher() i=5 2023-09-22T13:09:46.828 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.827+0000 7f1be2a14500 20 remove_watcher() i=6 2023-09-22T13:09:46.828 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.828+0000 7f1be2a14500 20 remove_watcher() i=7 2023-09-22T13:09:46.829 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.829+0000 7f1b9f7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.835 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:09:46.859 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:46.859 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:46.867 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.866+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.868 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.868+0000 7f3c33853500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.868 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.868+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.869 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.868+0000 7f3c33853500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.869 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.868+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.869 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.869+0000 7f3c33853500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.870 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.869+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.870 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.869+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.880 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.878+0000 7f3c33853500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:46.880 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.878+0000 7f3c33853500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:46.880 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.878+0000 7f3c33853500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:46.880 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.878+0000 7f3c33853500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:46.880 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.878+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.880 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.879+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.880 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.879+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.880 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.879+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.881 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.879+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.881 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.879+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.881 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.879+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.882 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.880+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.882 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.880+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.882 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.880+0000 7f3c33853500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.882 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.880+0000 7f3c33853500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:46.882 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.880+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.882 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.880+0000 7f3c33853500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.882 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.880+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.883 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.880+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.883 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.880+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.883 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.881+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.883 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.881+0000 7f3c33853500 20 zonegroup default 2023-09-22T13:09:46.883 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.881+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.883 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.881+0000 7f3c33853500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.883 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.881+0000 7f3c33853500 10 Cannot find current period zone using local zone 2023-09-22T13:09:46.883 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.881+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.883 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.881+0000 7f3c33853500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.883 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.881+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.884 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.881+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.884 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.881+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.884 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.882+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.885 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.882+0000 7f3c33853500 20 zone default found 2023-09-22T13:09:46.885 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.882+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.885 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.882+0000 7f3c33853500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.885 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.882+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.885 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.882+0000 7f3c33853500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.885 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.882+0000 7f3c33853500 20 started sync module instance, tier type = 2023-09-22T13:09:46.885 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.882+0000 7f3c33853500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:46.889 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.889+0000 7f3c33853500 20 add_watcher() i=0 2023-09-22T13:09:46.890 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.889+0000 7f3c33853500 20 add_watcher() i=1 2023-09-22T13:09:46.890 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.889+0000 7f3c33853500 20 add_watcher() i=2 2023-09-22T13:09:46.890 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.889+0000 7f3c33853500 20 add_watcher() i=3 2023-09-22T13:09:46.890 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.889+0000 7f3c33853500 20 add_watcher() i=4 2023-09-22T13:09:46.890 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.889+0000 7f3c33853500 20 add_watcher() i=5 2023-09-22T13:09:46.890 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.889+0000 7f3c33853500 20 add_watcher() i=6 2023-09-22T13:09:46.890 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.889+0000 7f3c33853500 20 add_watcher() i=7 2023-09-22T13:09:46.891 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.889+0000 7f3c33853500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:46.891 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.890+0000 7f3c1f536700 0 -- 172.21.15.98:0/4247292193 >> [v2:172.21.15.169:6821/140771,v1:172.21.15.169:6823/140771] conn(0x7f3c10027ae0 msgr2=0x7f3c10025440 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:46.892 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.891+0000 7f3beffe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.892 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.891+0000 7f3c33853500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:46.892 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.891+0000 7f3c33853500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:46.896 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.896+0000 7f3c1f536700 0 -- 172.21.15.98:0/3097439311 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x5613830eee10 msgr2=0x5613831075d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:46.899 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.898+0000 7f3c33853500 5 note: GC not initialized 2023-09-22T13:09:46.900 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.899+0000 7f3b5effd700 20 reqs_thread_entry: start 2023-09-22T13:09:46.901 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.899+0000 7f3b5cff9700 20 reqs_thread_entry: start 2023-09-22T13:09:46.901 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.899+0000 7f3c33853500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:46.901 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.899+0000 7f3c33853500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:46.901 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.899+0000 7f3b52ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:17 2023 2023-09-22T13:09:46.901 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.901 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.899+0000 7f3b527fc700 20 reqs_thread_entry: start 2023-09-22T13:09:46.901 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.899+0000 7f3c33853500 20 get_system_obj_state: rctx=0x7ffcc1ea25b0 obj=default.rgw.meta:root:check-bucket state=0x561382fe4980 s->prefetch_data=0 2023-09-22T13:09:46.901 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.900+0000 7f3c33853500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.901 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.900+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.902 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.900+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:46.903 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.900+0000 7f3c33853500 20 get_system_obj_state: rctx=0x7ffcc1ea25b0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x561382f43810 s->prefetch_data=0 2023-09-22T13:09:46.903 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.903 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:46.903 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:46.903 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.903 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:46.903 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:46.903 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 get_system_obj_state: rctx=0x7ffcc1ea2380 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x561382f43810 s->prefetch_data=0 2023-09-22T13:09:46.903 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.903 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:46.903 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:46.904 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.904 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.901+0000 7f3c33853500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:46.904 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.902+0000 7f3c33853500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:46.904 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.902+0000 7f3c33853500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.904 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.904+0000 7f3c33853500 20 remove_watcher() i=0 2023-09-22T13:09:46.904 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.904+0000 7f3c33853500 2 removed watcher, disabling cache 2023-09-22T13:09:46.905 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.904+0000 7f3c33853500 20 remove_watcher() i=1 2023-09-22T13:09:46.906 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.905+0000 7f3c33853500 20 remove_watcher() i=2 2023-09-22T13:09:46.906 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.906+0000 7f3c33853500 20 remove_watcher() i=3 2023-09-22T13:09:46.907 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.907+0000 7f3c33853500 20 remove_watcher() i=4 2023-09-22T13:09:46.908 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.907+0000 7f3c33853500 20 remove_watcher() i=5 2023-09-22T13:09:46.909 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.908+0000 7f3c33853500 20 remove_watcher() i=6 2023-09-22T13:09:46.910 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.909+0000 7f3c33853500 20 remove_watcher() i=7 2023-09-22T13:09:46.910 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.910+0000 7f3beffe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.915 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: bucket check olh finds leftover OLH entries 2023-09-22T13:09:46.915 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.916 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bucket check olh --bucket check-bucket --dump-keys 2023-09-22T13:09:46.940 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:46.941 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:46.948 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.948+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.950 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.949+0000 7f5dae674500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.950 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.949+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.951 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.950+0000 7f5dae674500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.951 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.950+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.951 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.950+0000 7f5dae674500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.951 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.950+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.951 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.951+0000 7f5dae674500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.961 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.960+0000 7f5dae674500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:46.961 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.960+0000 7f5dae674500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:46.961 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.960+0000 7f5dae674500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:46.961 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.960+0000 7f5dae674500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:46.961 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.960+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.962 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.960+0000 7f5dae674500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.962 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.960+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.962 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.960+0000 7f5dae674500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.962 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.961+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.964 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.961+0000 7f5dae674500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.964 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.961+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.964 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.961+0000 7f5dae674500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.964 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.961+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.964 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.962+0000 7f5dae674500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.964 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.962+0000 7f5dae674500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.962+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.962+0000 7f5dae674500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.962+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.962+0000 7f5dae674500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.962+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.962+0000 7f5dae674500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.962+0000 7f5dae674500 20 zonegroup default 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.962+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 10 Cannot find current period zone using local zone 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.965 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.966 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:46.966 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.966 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:46.966 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 20 zone default found 2023-09-22T13:09:46.966 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.966 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.966 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.963+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.966 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.964+0000 7f5dae674500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:46.967 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.964+0000 7f5dae674500 20 started sync module instance, tier type = 2023-09-22T13:09:46.967 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.964+0000 7f5dae674500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:46.970 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.970+0000 7f5dae674500 20 add_watcher() i=0 2023-09-22T13:09:46.971 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.970+0000 7f5dae674500 20 add_watcher() i=1 2023-09-22T13:09:46.971 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.970+0000 7f5dae674500 20 add_watcher() i=2 2023-09-22T13:09:46.971 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.970+0000 7f5dae674500 20 add_watcher() i=3 2023-09-22T13:09:46.971 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.970+0000 7f5dae674500 20 add_watcher() i=4 2023-09-22T13:09:46.971 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.970+0000 7f5dae674500 20 add_watcher() i=5 2023-09-22T13:09:46.971 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.970+0000 7f5dae674500 20 add_watcher() i=6 2023-09-22T13:09:46.971 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.970+0000 7f5dae674500 20 add_watcher() i=7 2023-09-22T13:09:46.972 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.970+0000 7f5dae674500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:46.972 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.972+0000 7f5d6b7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.973 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.972+0000 7f5dae674500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:46.973 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.972+0000 7f5dae674500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:46.974 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.973+0000 7f5d9a357700 0 -- >> [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] conn(0x561934116700 msgr2=0x56193411cc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2023-09-22T13:09:46.978 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.978+0000 7f5dae674500 5 note: GC not initialized 2023-09-22T13:09:46.980 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.979+0000 7f5cda7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:46.980 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.979+0000 7f5ccbfff700 20 reqs_thread_entry: start 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.979+0000 7f5dae674500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.979+0000 7f5dae674500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.979+0000 7f5cca7fc700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:17 2023 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.979+0000 7f5cc9ffb700 20 reqs_thread_entry: start 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.979+0000 7f5dae674500 20 get_system_obj_state: rctx=0x7ffd8ff68068 obj=default.rgw.meta:root:check-bucket state=0x561934006000 s->prefetch_data=0 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.979+0000 7f5dae674500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5dae674500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5dae674500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5dae674500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5dae674500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.981 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5dae674500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:09:46.982 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.983 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5dae674500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:46.983 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5dae674500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:09:46.983 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5dae674500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:46.983 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5dae674500 20 get_system_obj_state: rctx=0x7ffd8ff68138 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x561934006000 s->prefetch_data=0 2023-09-22T13:09:46.983 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5dae674500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : miss 2023-09-22T13:09:46.983 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.980+0000 7f5d99b56700 0 -- 172.21.15.98:0/1152176128 >> [v2:172.21.15.98:6820/140891,v1:172.21.15.98:6822/140891] conn(0x7f5d8c0233f0 msgr2=0x7f5d8c023830 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:09:46.983 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x16 2023-09-22T13:09:46.983 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 20 rados->read ofs=0 len=0 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x13 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:46.984 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:46.985 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 2023-09-22T13:09:46.985 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.981+0000 7f5dae674500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:46.985 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.982+0000 7f5dae674500 1 NOTICE: finished shard 0 (2 entries found) 2023-09-22T13:09:46.985 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.982+0000 7f5dae674500 1 NOTICE: finished all shards (2 entries found) 2023-09-22T13:09:46.985 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.983+0000 7f5dae674500 20 remove_watcher() i=0 2023-09-22T13:09:46.985 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.983+0000 7f5dae674500 2 removed watcher, disabling cache 2023-09-22T13:09:46.986 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.984+0000 7f5dae674500 20 remove_watcher() i=1 2023-09-22T13:09:46.986 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.985+0000 7f5dae674500 20 remove_watcher() i=2 2023-09-22T13:09:46.986 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.985+0000 7f5dae674500 20 remove_watcher() i=3 2023-09-22T13:09:46.987 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.986+0000 7f5dae674500 20 remove_watcher() i=4 2023-09-22T13:09:46.988 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.987+0000 7f5dae674500 20 remove_watcher() i=5 2023-09-22T13:09:46.988 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.988+0000 7f5dae674500 20 remove_watcher() i=6 2023-09-22T13:09:46.989 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.988+0000 7f5dae674500 20 remove_watcher() i=7 2023-09-22T13:09:46.990 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:46.989+0000 7f5d6b7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:46.994 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: bucket check olh fixes leftover OLH entries 2023-09-22T13:09:46.994 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:46.995 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bucket check olh --bucket check-bucket --fix --rgw-olh-pending-timeout-sec 0 --dump-keys 2023-09-22T13:09:47.022 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:47.022 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:47.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.029+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.031 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.030+0000 7f326d7e2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.031 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.030+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.031+0000 7f326d7e2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.031+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.031+0000 7f326d7e2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.031+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.032+0000 7f326d7e2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.042 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.041+0000 7f326d7e2500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:47.042 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.041+0000 7f326d7e2500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:47.042 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.041+0000 7f326d7e2500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:47.042 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.041+0000 7f326d7e2500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:47.042 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.041+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.042 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.041+0000 7f326d7e2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.042 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.041+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.043 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.041+0000 7f326d7e2500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:47.043 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.041+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.044 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.042+0000 7f326d7e2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.044 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.042+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.044 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.042+0000 7f326d7e2500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.042+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.042+0000 7f326d7e2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.042+0000 7f326d7e2500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.042+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.042+0000 7f326d7e2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.042+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.043+0000 7f326d7e2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.043+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.043+0000 7f326d7e2500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.043+0000 7f326d7e2500 20 zonegroup default 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.043+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.045 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.043+0000 7f326d7e2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.046 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.043+0000 7f326d7e2500 10 Cannot find current period zone using local zone 2023-09-22T13:09:47.046 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.043+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.046 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.043+0000 7f326d7e2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.046 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.043+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.046 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.044+0000 7f326d7e2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.046 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.044+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.047 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.044+0000 7f326d7e2500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:47.047 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.044+0000 7f326d7e2500 20 zone default found 2023-09-22T13:09:47.047 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.044+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.047 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.044+0000 7f326d7e2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.047 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.044+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.047 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.045+0000 7f326d7e2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.047 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.045+0000 7f326d7e2500 20 started sync module instance, tier type = 2023-09-22T13:09:47.047 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.045+0000 7f326d7e2500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:47.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.051+0000 7f326d7e2500 20 add_watcher() i=0 2023-09-22T13:09:47.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.051+0000 7f326d7e2500 20 add_watcher() i=1 2023-09-22T13:09:47.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.051+0000 7f326d7e2500 20 add_watcher() i=2 2023-09-22T13:09:47.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.051+0000 7f326d7e2500 20 add_watcher() i=3 2023-09-22T13:09:47.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.051+0000 7f326d7e2500 20 add_watcher() i=4 2023-09-22T13:09:47.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.051+0000 7f326d7e2500 20 add_watcher() i=5 2023-09-22T13:09:47.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.051+0000 7f326d7e2500 20 add_watcher() i=6 2023-09-22T13:09:47.052 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.051+0000 7f326d7e2500 20 add_watcher() i=7 2023-09-22T13:09:47.053 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.051+0000 7f326d7e2500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:47.053 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.053+0000 7f3229fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:47.054 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.053+0000 7f326d7e2500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:47.054 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.053+0000 7f326d7e2500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:47.055 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.054+0000 7f3253fff700 0 -- >> [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] conn(0x5652dab69bf0 msgr2=0x5652dab6a8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2023-09-22T13:09:47.059 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.059+0000 7f326d7e2500 5 note: GC not initialized 2023-09-22T13:09:47.061 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.060+0000 7f31a0ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:47.061 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.060+0000 7f31967fc700 20 reqs_thread_entry: start 2023-09-22T13:09:47.061 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.060+0000 7f326d7e2500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:47.061 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.060+0000 7f326d7e2500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:47.061 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.060+0000 7f3194ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:17 2023 2023-09-22T13:09:47.061 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:47.062 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.060+0000 7f317bfff700 20 reqs_thread_entry: start 2023-09-22T13:09:47.062 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.060+0000 7f326d7e2500 20 get_system_obj_state: rctx=0x7fffe7bea1d8 obj=default.rgw.meta:root:check-bucket state=0x5652dab6cad0 s->prefetch_data=0 2023-09-22T13:09:47.062 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.060+0000 7f326d7e2500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:09:47.063 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:09:47.064 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:47.064 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:47.064 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:47.064 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:09:47.064 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.064 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f3253fff700 0 -- 172.21.15.98:0/4083555800 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f324c02b350 msgr2=0x7f324c04cfe0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:47.064 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:47.064 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:09:47.064 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:47.064 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 20 get_system_obj_state: rctx=0x7fffe7bea2a8 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x5652dab6cad0 s->prefetch_data=0 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : miss 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x16 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.061+0000 7f326d7e2500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.062+0000 7f326d7e2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.062+0000 7f326d7e2500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:47.065 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.062+0000 7f326d7e2500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x13 2023-09-22T13:09:47.066 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.062+0000 7f326d7e2500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:47.066 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.062+0000 7f326d7e2500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:47.066 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.062+0000 7f326d7e2500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:47.066 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.062+0000 7f326d7e2500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 2023-09-22T13:09:47.066 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.062+0000 7f326d7e2500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:47.066 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.063+0000 7f326d7e2500 20 get_obj_state: rctx=0x5652dac14680 obj=check-bucket:c state=0x5652dac15298 s->prefetch_data=0 2023-09-22T13:09:47.066 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.063+0000 7f326d7e2500 20 get_obj_state: setting s->obj_tag to fwz1kdf2exrv8oz808vcd86flttnp9y9 2023-09-22T13:09:47.066 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.063+0000 7f326d7e2500 20 get_obj_state_impl: setting s->olh_tag to lsnbnq1g01z87mbv2lynjdm52bc7u3pd 2023-09-22T13:09:47.066 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.063+0000 7f326d7e2500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:47.066 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.063+0000 7f326d7e2500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:47.067 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.063+0000 7f326d7e2500 20 olh_log_entry: epoch=4 op=2 key=c[] 2023-09-22T13:09:47.067 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.063+0000 7f326d7e2500 20 olh_log_entry: epoch=4 op=3 key=c[HXtM1JVdoo0aUZ4RpcGfzmJJ4DUR8ec] 2023-09-22T13:09:47.067 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.063+0000 7f326d7e2500 20 get_obj_state: rctx=0x5652dac14680 obj=check-bucket:_:HXtM1JVdoo0aUZ4RpcGfzmJJ4DUR8ec_c state=0x5652dac1ea38 s->prefetch_data=0 2023-09-22T13:09:47.067 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.065+0000 7f326d7e2500 20 get_obj_state: rctx=0x5652dac14680 obj=check-bucket:c state=0x5652dac15298 s->prefetch_data=0 2023-09-22T13:09:47.067 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.065+0000 7f326d7e2500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=lsnbnq1g01z87mbv2lynjdm52bc7u3pd 2023-09-22T13:09:47.068 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.067+0000 7f326d7e2500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:47.069 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.068+0000 7f326d7e2500 20 get_obj_state: rctx=0x5652dac14680 obj=check-bucket:d state=0x5652dac17948 s->prefetch_data=0 2023-09-22T13:09:47.070 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.068+0000 7f326d7e2500 20 get_obj_state: setting s->obj_tag to 9qz07ey9ho4ol5rvep5dza03rtzr38wy 2023-09-22T13:09:47.070 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.068+0000 7f326d7e2500 20 get_obj_state_impl: setting s->olh_tag to 74fro5yhmgfgyk5mjfl5mb630yt3fn9j 2023-09-22T13:09:47.070 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.068+0000 7f326d7e2500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:47.070 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.068+0000 7f326d7e2500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:47.070 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.069+0000 7f326d7e2500 20 olh_log_entry: epoch=4 op=2 key=d[] 2023-09-22T13:09:47.070 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.069+0000 7f326d7e2500 20 olh_log_entry: epoch=4 op=3 key=d[NYDHzQtzOrTI.9W6HwA6ahiHguZRw0t] 2023-09-22T13:09:47.070 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.069+0000 7f326d7e2500 20 get_obj_state: rctx=0x5652dac14680 obj=check-bucket:_:NYDHzQtzOrTI.9W6HwA6ahiHguZRw0t_d state=0x5652dac1f6f8 s->prefetch_data=0 2023-09-22T13:09:47.071 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.070+0000 7f326d7e2500 20 get_obj_state: rctx=0x5652dac14680 obj=check-bucket:d state=0x5652dac17948 s->prefetch_data=0 2023-09-22T13:09:47.071 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.070+0000 7f326d7e2500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=74fro5yhmgfgyk5mjfl5mb630yt3fn9j 2023-09-22T13:09:47.073 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.072+0000 7f326d7e2500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:47.074 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.073+0000 7f326d7e2500 1 NOTICE: finished shard 0 (2 entries removed) 2023-09-22T13:09:47.074 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.073+0000 7f326d7e2500 1 NOTICE: finished all shards (2 entries removed) 2023-09-22T13:09:47.075 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.074+0000 7f326d7e2500 20 remove_watcher() i=0 2023-09-22T13:09:47.075 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.074+0000 7f326d7e2500 2 removed watcher, disabling cache 2023-09-22T13:09:47.075 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.075+0000 7f326d7e2500 20 remove_watcher() i=1 2023-09-22T13:09:47.076 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.076+0000 7f326d7e2500 20 remove_watcher() i=2 2023-09-22T13:09:47.077 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.076+0000 7f326d7e2500 20 remove_watcher() i=3 2023-09-22T13:09:47.078 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.077+0000 7f326d7e2500 20 remove_watcher() i=4 2023-09-22T13:09:47.079 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.078+0000 7f326d7e2500 20 remove_watcher() i=5 2023-09-22T13:09:47.079 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.079+0000 7f326d7e2500 20 remove_watcher() i=6 2023-09-22T13:09:47.080 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.079+0000 7f326d7e2500 20 remove_watcher() i=7 2023-09-22T13:09:47.081 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.080+0000 7f3229fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:47.087 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:09:47.111 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:47.111 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:47.119 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.119+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.121 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.120+0000 7f0f7cc2e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.121 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.120+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.121 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.121+0000 7f0f7cc2e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.122 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.121+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.122 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.121+0000 7f0f7cc2e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.122 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.121+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.123 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.122+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.133 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.131+0000 7f0f7cc2e500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:47.133 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.131+0000 7f0f7cc2e500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:47.133 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.131+0000 7f0f7cc2e500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:47.133 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.131+0000 7f0f7cc2e500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:47.133 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.131+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.133 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.132+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.133 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.132+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.134 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.132+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:47.134 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.132+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.134 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.132+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.134 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.132+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.135 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.133+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.133+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.133+0000 7f0f7cc2e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.133+0000 7f0f7cc2e500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.133+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.133+0000 7f0f7cc2e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.133+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.133+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.133+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.134+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.134+0000 7f0f7cc2e500 20 zonegroup default 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.134+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.136 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.134+0000 7f0f7cc2e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.134+0000 7f0f7cc2e500 10 Cannot find current period zone using local zone 2023-09-22T13:09:47.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.134+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.134+0000 7f0f7cc2e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.134+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.134+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.134+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.135+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:47.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.135+0000 7f0f7cc2e500 20 zone default found 2023-09-22T13:09:47.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.135+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.135+0000 7f0f7cc2e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.135+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.138 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.135+0000 7f0f7cc2e500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.138 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.135+0000 7f0f7cc2e500 20 started sync module instance, tier type = 2023-09-22T13:09:47.138 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.135+0000 7f0f7cc2e500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:47.143 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.142+0000 7f0f6790f700 0 -- 172.21.15.98:0/256631503 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f0f5802b960 msgr2=0x7f0f580283c0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:47.143 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.142+0000 7f0f7cc2e500 20 add_watcher() i=0 2023-09-22T13:09:47.143 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.142+0000 7f0f7cc2e500 20 add_watcher() i=1 2023-09-22T13:09:47.143 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.142+0000 7f0f7cc2e500 20 add_watcher() i=2 2023-09-22T13:09:47.143 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.142+0000 7f0f7cc2e500 20 add_watcher() i=3 2023-09-22T13:09:47.143 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.142+0000 7f0f7cc2e500 20 add_watcher() i=4 2023-09-22T13:09:47.143 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.142+0000 7f0f7cc2e500 20 add_watcher() i=5 2023-09-22T13:09:47.143 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.142+0000 7f0f7cc2e500 20 add_watcher() i=6 2023-09-22T13:09:47.144 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.143+0000 7f0f7cc2e500 20 add_watcher() i=7 2023-09-22T13:09:47.144 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.143+0000 7f0f7cc2e500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:47.145 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.144+0000 7f0f39fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:47.145 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.144+0000 7f0f7cc2e500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:47.145 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.144+0000 7f0f7cc2e500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:47.148 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.148+0000 7f0f68110700 0 -- 172.21.15.98:0/241381893 >> [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] conn(0x55e68afa6750 msgr2=0x55e68af9d7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:47.151 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.150+0000 7f0f7cc2e500 5 note: GC not initialized 2023-09-22T13:09:47.152 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.151+0000 7f0eb0ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:47.152 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.151+0000 7f0ea67fc700 20 reqs_thread_entry: start 2023-09-22T13:09:47.152 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.151+0000 7f0f7cc2e500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:47.153 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.151+0000 7f0f7cc2e500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:47.153 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.151+0000 7f0ea4ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:17 2023 2023-09-22T13:09:47.153 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:47.153 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.151+0000 7f0e8bfff700 20 reqs_thread_entry: start 2023-09-22T13:09:47.153 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.151+0000 7f0f7cc2e500 20 get_system_obj_state: rctx=0x7ffec9c3eb50 obj=default.rgw.meta:root:check-bucket state=0x55e68af16f50 s->prefetch_data=0 2023-09-22T13:09:47.153 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.152+0000 7f0f7cc2e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:47.153 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.152+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.154 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.152+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:47.154 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.152+0000 7f0f7cc2e500 20 get_system_obj_state: rctx=0x7ffec9c3eb50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55e68ae98000 s->prefetch_data=0 2023-09-22T13:09:47.155 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.152+0000 7f0f7cc2e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:47.155 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.152+0000 7f0f7cc2e500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:47.155 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.152+0000 7f0f7cc2e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:47.155 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.152+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.155 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.153+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:47.155 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.153+0000 7f0f7cc2e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:47.155 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.153+0000 7f0f7cc2e500 20 get_system_obj_state: rctx=0x7ffec9c3e920 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55e68ae98000 s->prefetch_data=0 2023-09-22T13:09:47.155 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.153+0000 7f0f7cc2e500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:47.155 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.153+0000 7f0f7cc2e500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:47.156 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.153+0000 7f0f7cc2e500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:47.156 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.153+0000 7f0f7cc2e500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.156 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.153+0000 7f0f7cc2e500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:47.156 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.153+0000 7f0f7cc2e500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:47.156 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.153+0000 7f0f7cc2e500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:47.156 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.155+0000 7f0f7cc2e500 20 remove_watcher() i=0 2023-09-22T13:09:47.156 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.155+0000 7f0f7cc2e500 2 removed watcher, disabling cache 2023-09-22T13:09:47.157 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.156+0000 7f0f7cc2e500 20 remove_watcher() i=1 2023-09-22T13:09:47.157 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.156+0000 7f0f7cc2e500 20 remove_watcher() i=2 2023-09-22T13:09:47.158 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.157+0000 7f0f7cc2e500 20 remove_watcher() i=3 2023-09-22T13:09:47.159 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.158+0000 7f0f7cc2e500 20 remove_watcher() i=4 2023-09-22T13:09:47.160 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.159+0000 7f0f7cc2e500 20 remove_watcher() i=5 2023-09-22T13:09:47.160 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.160+0000 7f0f7cc2e500 20 remove_watcher() i=6 2023-09-22T13:09:47.161 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.161+0000 7f0f7cc2e500 20 remove_watcher() i=7 2023-09-22T13:09:47.162 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.161+0000 7f0f39fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:47.168 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:09:47.192 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:47.192 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:47.401 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.400+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.403 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.402+0000 7f4731b27500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.403 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.402+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.404 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.403+0000 7f4731b27500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.404 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.403+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.404 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.403+0000 7f4731b27500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.404 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.403+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.404 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.404+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.415 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.413+0000 7f4731b27500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:47.415 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.413+0000 7f4731b27500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:47.415 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.413+0000 7f4731b27500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:47.415 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.413+0000 7f4731b27500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:47.415 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.413+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.415 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.414+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.415 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.414+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.415 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.414+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:47.415 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.414+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.416 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.414+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.416 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.414+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.417 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.415+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:47.417 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.415+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.417 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.415+0000 7f4731b27500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.417 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.415+0000 7f4731b27500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:47.418 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.415+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.418 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.415+0000 7f4731b27500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.418 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.415+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.418 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.415+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.418 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.415+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.418 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:47.418 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 zonegroup default 2023-09-22T13:09:47.418 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.418 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.418 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 10 Cannot find current period zone using local zone 2023-09-22T13:09:47.418 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.419 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.419 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.419 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.419 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.419 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:47.419 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 zone default found 2023-09-22T13:09:47.419 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.416+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.419 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.417+0000 7f4731b27500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.419 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.417+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.420 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.417+0000 7f4731b27500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.420 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.417+0000 7f4731b27500 20 started sync module instance, tier type = 2023-09-22T13:09:47.420 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.417+0000 7f4731b27500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:47.423 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.422+0000 7f471c808700 0 -- 172.21.15.98:0/1600731858 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f4714026920 msgr2=0x7f47140253a0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:47.425 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.424+0000 7f4731b27500 20 add_watcher() i=0 2023-09-22T13:09:47.425 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.424+0000 7f4731b27500 20 add_watcher() i=1 2023-09-22T13:09:47.425 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.424+0000 7f4731b27500 20 add_watcher() i=2 2023-09-22T13:09:47.425 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.424+0000 7f4731b27500 20 add_watcher() i=3 2023-09-22T13:09:47.425 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.424+0000 7f4731b27500 20 add_watcher() i=4 2023-09-22T13:09:47.425 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.424+0000 7f4731b27500 20 add_watcher() i=5 2023-09-22T13:09:47.425 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.424+0000 7f4731b27500 20 add_watcher() i=6 2023-09-22T13:09:47.426 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.424+0000 7f4731b27500 20 add_watcher() i=7 2023-09-22T13:09:47.426 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.424+0000 7f4731b27500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:47.426 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.426+0000 7f46ee7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:47.427 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.426+0000 7f4731b27500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:47.427 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.426+0000 7f4731b27500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:47.430 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.430+0000 7f471d80a700 0 -- 172.21.15.98:0/2058080091 >> [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] conn(0x563fed16d5e0 msgr2=0x563fed1700c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:47.432 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.432+0000 7f4731b27500 5 note: GC not initialized 2023-09-22T13:09:47.434 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.432+0000 7f465d7fa700 20 reqs_thread_entry: start 2023-09-22T13:09:47.434 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.433+0000 7f464effd700 20 reqs_thread_entry: start 2023-09-22T13:09:47.434 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.433+0000 7f4731b27500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:47.434 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.433+0000 7f4731b27500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:47.434 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.433+0000 7f464d7fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:17 2023 2023-09-22T13:09:47.434 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:47.434 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.433+0000 7f464cff9700 20 reqs_thread_entry: start 2023-09-22T13:09:47.434 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.433+0000 7f4731b27500 20 get_system_obj_state: rctx=0x7ffc6eb7ba90 obj=default.rgw.meta:root:check-bucket state=0x563fed060980 s->prefetch_data=0 2023-09-22T13:09:47.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.433+0000 7f4731b27500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:47.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.433+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.434+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:47.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.434+0000 7f4731b27500 20 get_system_obj_state: rctx=0x7ffc6eb7ba90 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x563fecfbf810 s->prefetch_data=0 2023-09-22T13:09:47.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.434+0000 7f4731b27500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:47.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.434+0000 7f4731b27500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:47.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.434+0000 7f4731b27500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:47.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.434+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.435+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:47.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.435+0000 7f4731b27500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:47.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.435+0000 7f4731b27500 20 get_system_obj_state: rctx=0x7ffc6eb7b860 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x563fecfbf810 s->prefetch_data=0 2023-09-22T13:09:47.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.435+0000 7f4731b27500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:47.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.435+0000 7f4731b27500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:47.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.435+0000 7f4731b27500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:47.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.435+0000 7f4731b27500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.435+0000 7f4731b27500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:47.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.435+0000 7f4731b27500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:47.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.435+0000 7f4731b27500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:47.438 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.437+0000 7f4731b27500 20 remove_watcher() i=0 2023-09-22T13:09:47.438 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.437+0000 7f4731b27500 2 removed watcher, disabling cache 2023-09-22T13:09:47.438 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.438+0000 7f4731b27500 20 remove_watcher() i=1 2023-09-22T13:09:47.439 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.438+0000 7f4731b27500 20 remove_watcher() i=2 2023-09-22T13:09:47.440 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.439+0000 7f4731b27500 20 remove_watcher() i=3 2023-09-22T13:09:47.440 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.440+0000 7f4731b27500 20 remove_watcher() i=4 2023-09-22T13:09:47.441 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.441+0000 7f4731b27500 20 remove_watcher() i=5 2023-09-22T13:09:47.442 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.441+0000 7f4731b27500 20 remove_watcher() i=6 2023-09-22T13:09:47.443 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.442+0000 7f4731b27500 20 remove_watcher() i=7 2023-09-22T13:09:47.444 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.443+0000 7f46ee7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:47.448 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: bucket check olh does not find new leftover OLH entries after fix 2023-09-22T13:09:47.448 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:47.450 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bucket check olh --bucket check-bucket --dump-keys 2023-09-22T13:09:47.474 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:47.474 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:47.482 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.482+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.484 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.483+0000 7fd9b6fb6500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.484 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.483+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.484 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.484+0000 7fd9b6fb6500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.484+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.484+0000 7fd9b6fb6500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.484+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.485 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.485+0000 7fd9b6fb6500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.496 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.494+0000 7fd9b6fb6500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:47.496 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.494+0000 7fd9b6fb6500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:47.496 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.494+0000 7fd9b6fb6500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:47.496 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.494+0000 7fd9b6fb6500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:47.496 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.494+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.496 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.494+0000 7fd9b6fb6500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.496 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.494+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.496 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.495+0000 7fd9b6fb6500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:47.496 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.495+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.496 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.495+0000 7fd9b6fb6500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.496 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.495+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.498 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.495+0000 7fd9b6fb6500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:47.498 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.495+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.498 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.496+0000 7fd9b6fb6500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.498 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.496+0000 7fd9b6fb6500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.496+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.496+0000 7fd9b6fb6500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.496+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.496+0000 7fd9b6fb6500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.496+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.496+0000 7fd9b6fb6500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.496+0000 7fd9b6fb6500 20 zonegroup default 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.496+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.497+0000 7fd9b6fb6500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.497+0000 7fd9b6fb6500 10 Cannot find current period zone using local zone 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.497+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.497+0000 7fd9b6fb6500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.497+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.497+0000 7fd9b6fb6500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:47.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.497+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.497+0000 7fd9b6fb6500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:47.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.497+0000 7fd9b6fb6500 20 zone default found 2023-09-22T13:09:47.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.497+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.498+0000 7fd9b6fb6500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.498+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.498+0000 7fd9b6fb6500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:47.501 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.498+0000 7fd9b6fb6500 20 started sync module instance, tier type = 2023-09-22T13:09:47.501 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.498+0000 7fd9b6fb6500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:47.504 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.503+0000 7fd9a1c97700 0 -- 172.21.15.98:0/1272785515 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7fd998020960 msgr2=0x7fd99801f440 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:47.506 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.505+0000 7fd9b6fb6500 20 add_watcher() i=0 2023-09-22T13:09:47.506 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.505+0000 7fd9b6fb6500 20 add_watcher() i=1 2023-09-22T13:09:47.506 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.505+0000 7fd9b6fb6500 20 add_watcher() i=2 2023-09-22T13:09:47.506 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.505+0000 7fd9b6fb6500 20 add_watcher() i=3 2023-09-22T13:09:47.506 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.505+0000 7fd9b6fb6500 20 add_watcher() i=4 2023-09-22T13:09:47.506 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.505+0000 7fd9b6fb6500 20 add_watcher() i=5 2023-09-22T13:09:47.506 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.505+0000 7fd9b6fb6500 20 add_watcher() i=6 2023-09-22T13:09:47.507 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.506+0000 7fd9b6fb6500 20 add_watcher() i=7 2023-09-22T13:09:47.507 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.506+0000 7fd9b6fb6500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:47.508 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.507+0000 7fd973fdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:47.508 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.508+0000 7fd9b6fb6500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:47.508 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.508+0000 7fd9b6fb6500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:47.512 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.512+0000 7fd9a2498700 0 -- 172.21.15.98:0/1640563169 >> [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] conn(0x55b2c2ded670 msgr2=0x55b2c2dff520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:47.514 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.513+0000 7fd9b6fb6500 5 note: GC not initialized 2023-09-22T13:09:47.516 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.514+0000 7fd8e2ffd700 20 reqs_thread_entry: start 2023-09-22T13:09:47.516 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.515+0000 7fd8e0ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:47.516 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.515+0000 7fd9b6fb6500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:47.516 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.515+0000 7fd9b6fb6500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:47.516 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.515+0000 7fd8d6ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:17 2023 2023-09-22T13:09:47.516 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:47.516 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.515+0000 7fd9b6fb6500 20 get_system_obj_state: rctx=0x7fff7c81a1b8 obj=default.rgw.meta:root:check-bucket state=0x55b2c2d575b0 s->prefetch_data=0 2023-09-22T13:09:47.516 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.515+0000 7fd8d67fc700 20 reqs_thread_entry: start 2023-09-22T13:09:47.516 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.515+0000 7fd9b6fb6500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:09:47.518 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:09:47.518 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:47.518 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:47.518 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:47.518 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:09:47.518 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.518 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:47.518 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:09:47.518 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 20 get_system_obj_state: rctx=0x7fff7c81a288 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55b2c2d575b0 s->prefetch_data=0 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : miss 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x16 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.516+0000 7fd9b6fb6500 20 rados->read ofs=0 len=0 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.517+0000 7fd9b6fb6500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:47.519 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.517+0000 7fd9b6fb6500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x13 2023-09-22T13:09:47.520 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.517+0000 7fd9b6fb6500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:47.520 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.517+0000 7fd9b6fb6500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:47.520 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.517+0000 7fd9b6fb6500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:47.520 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.517+0000 7fd9b6fb6500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 2023-09-22T13:09:47.520 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.517+0000 7fd9b6fb6500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:47.520 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.517+0000 7fd9b6fb6500 1 NOTICE: finished shard 0 (0 entries found) 2023-09-22T13:09:47.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.517+0000 7fd9b6fb6500 1 NOTICE: finished all shards (0 entries found) 2023-09-22T13:09:47.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.519+0000 7fd9b6fb6500 20 remove_watcher() i=0 2023-09-22T13:09:47.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.519+0000 7fd9b6fb6500 2 removed watcher, disabling cache 2023-09-22T13:09:47.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.520+0000 7fd9b6fb6500 20 remove_watcher() i=1 2023-09-22T13:09:47.521 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.520+0000 7fd9b6fb6500 20 remove_watcher() i=2 2023-09-22T13:09:47.522 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.521+0000 7fd9b6fb6500 20 remove_watcher() i=3 2023-09-22T13:09:47.523 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.522+0000 7fd9b6fb6500 20 remove_watcher() i=4 2023-09-22T13:09:47.523 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.523+0000 7fd9b6fb6500 20 remove_watcher() i=5 2023-09-22T13:09:47.524 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.523+0000 7fd9b6fb6500 20 remove_watcher() i=6 2023-09-22T13:09:47.525 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.524+0000 7fd9b6fb6500 20 remove_watcher() i=7 2023-09-22T13:09:47.525 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:47.525+0000 7fd973fdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:47.530 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: verify that bucket check fixes do not affect null version objects 2023-09-22T13:09:47.530 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:47.581 INFO:tasks.workunit.client.0.smithi098.stderr:TEST: bucket check stats are correct in the presence of unlinked entries 2023-09-22T13:09:47.581 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:47.766 INFO:tasks.workunit.client.0.smithi098.stderr:ceph config set client rgw_debug_inject_set_olh_err 2 2023-09-22T13:09:48.160 INFO:tasks.workunit.client.0.smithi098.stderr:ceph config set client rgw_debug_inject_olh_cancel_modification_err true 2023-09-22T13:09:49.607 INFO:tasks.workunit.client.0.smithi098.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:09:49.609 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object c 2023-09-22T13:09:49.636 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:49.636 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:49.646 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.645+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.648 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.647+0000 7f269e911500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.648 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.647+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.649 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.648+0000 7f269e911500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.649 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.648+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.649 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.648+0000 7f269e911500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.649 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.649+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.650 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.649+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:49.661 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.660+0000 7f269e911500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:49.662 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.660+0000 7f269e911500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:49.662 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.660+0000 7f269e911500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:49.662 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.660+0000 7f269e911500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:49.662 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.660+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.662 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.661+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:49.662 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.661+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.663 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.661+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:49.663 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.661+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.663 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.661+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:49.663 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.661+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.664 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.662+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:49.664 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.662+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.664 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.662+0000 7f269e911500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.664 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.662+0000 7f269e911500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:49.664 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.662+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.664 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.662+0000 7f269e911500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.664 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.662+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.664 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.663+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:49.664 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.663+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.665 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.663+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:49.665 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.663+0000 7f269e911500 20 zonegroup default 2023-09-22T13:09:49.665 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.663+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.665 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.663+0000 7f269e911500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.666 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.663+0000 7f269e911500 10 Cannot find current period zone using local zone 2023-09-22T13:09:49.666 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.663+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.666 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.664+0000 7f269e911500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.666 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.664+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.666 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.664+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:49.666 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.664+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.666 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.664+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:49.666 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.664+0000 7f269e911500 20 zone default found 2023-09-22T13:09:49.666 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.664+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.666 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.664+0000 7f269e911500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.666 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.664+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.667 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.665+0000 7f269e911500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.667 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.665+0000 7f269e911500 20 started sync module instance, tier type = 2023-09-22T13:09:49.667 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.665+0000 7f269e911500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:49.673 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.673+0000 7f269e911500 20 add_watcher() i=0 2023-09-22T13:09:49.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.673+0000 7f269e911500 20 add_watcher() i=1 2023-09-22T13:09:49.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.673+0000 7f269e911500 20 add_watcher() i=2 2023-09-22T13:09:49.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.673+0000 7f269e911500 20 add_watcher() i=3 2023-09-22T13:09:49.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.673+0000 7f269e911500 20 add_watcher() i=4 2023-09-22T13:09:49.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.673+0000 7f269e911500 20 add_watcher() i=5 2023-09-22T13:09:49.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.673+0000 7f26895f2700 0 -- 172.21.15.98:0/1959677994 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f267c023a70 msgr2=0x7f267c0213d0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:09:49.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.673+0000 7f269e911500 20 add_watcher() i=6 2023-09-22T13:09:49.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.674+0000 7f269e911500 20 add_watcher() i=7 2023-09-22T13:09:49.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.674+0000 7f269e911500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:49.676 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.676+0000 7f265b7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:49.677 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.676+0000 7f269e911500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:49.677 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.676+0000 7f269e911500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:49.681 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.681+0000 7f2689df3700 0 -- 172.21.15.98:0/1774913951 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x56118dbf09b0 msgr2=0x56118dbca020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:49.683 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.683+0000 7f269e911500 5 note: GC not initialized 2023-09-22T13:09:49.685 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.684+0000 7f25ca7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:49.685 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.684+0000 7f25bbfff700 20 reqs_thread_entry: start 2023-09-22T13:09:49.685 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.684+0000 7f269e911500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:49.686 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.684+0000 7f269e911500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:49.686 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.684+0000 7f25ba7fc700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:20 2023 2023-09-22T13:09:49.686 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:49.686 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.684+0000 7f25b9ffb700 20 reqs_thread_entry: start 2023-09-22T13:09:49.686 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.685+0000 7f269e911500 20 get_system_obj_state: rctx=0x7ffed0babc50 obj=default.rgw.meta:root:check-bucket state=0x56118dbf6a00 s->prefetch_data=0 2023-09-22T13:09:49.686 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.685+0000 7f269e911500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:49.686 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.685+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.687 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.685+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:49.687 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.686+0000 7f269e911500 20 get_system_obj_state: rctx=0x7ffed0babc50 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x56118dbc8750 s->prefetch_data=0 2023-09-22T13:09:49.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.686+0000 7f269e911500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:49.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.686+0000 7f269e911500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:49.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.686+0000 7f269e911500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:49.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.686+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.686+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:49.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.687+0000 7f269e911500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:49.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.687+0000 7f269e911500 20 get_system_obj_state: rctx=0x7ffed0baba20 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x56118dbc8750 s->prefetch_data=0 2023-09-22T13:09:49.689 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.687+0000 7f269e911500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:49.689 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.687+0000 7f269e911500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:49.689 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.687+0000 7f269e911500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:49.689 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.687+0000 7f269e911500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.689 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.687+0000 7f269e911500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:49.689 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.687+0000 7f269e911500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:49.689 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.687+0000 7f269e911500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:49.690 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.689+0000 7f269e911500 20 remove_watcher() i=0 2023-09-22T13:09:49.690 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.689+0000 7f269e911500 2 removed watcher, disabling cache 2023-09-22T13:09:49.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.691+0000 7f269e911500 20 remove_watcher() i=1 2023-09-22T13:09:49.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.692+0000 7f269e911500 20 remove_watcher() i=2 2023-09-22T13:09:49.694 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.693+0000 7f269e911500 20 remove_watcher() i=3 2023-09-22T13:09:49.694 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.694+0000 7f269e911500 20 remove_watcher() i=4 2023-09-22T13:09:49.695 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.695+0000 7f269e911500 20 remove_watcher() i=5 2023-09-22T13:09:49.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.695+0000 7f269e911500 20 remove_watcher() i=6 2023-09-22T13:09:49.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.696+0000 7f269e911500 20 remove_watcher() i=7 2023-09-22T13:09:49.698 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.697+0000 7f265b7de700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:49.759 INFO:tasks.workunit.client.0.smithi098.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:09:49.760 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object d 2023-09-22T13:09:49.786 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:49.786 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:49.794 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.793+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.796 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.795+0000 7f22f1cbd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.796 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.795+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.797 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.796+0000 7f22f1cbd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.797 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.796+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.797 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.796+0000 7f22f1cbd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:49.797 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.796+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:49.798 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:49.797+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.011 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.009+0000 7f22f1cbd500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:50.011 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.009+0000 7f22f1cbd500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:50.011 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.009+0000 7f22f1cbd500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:50.011 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.009+0000 7f22f1cbd500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:50.011 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.009+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.011 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.009+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.011 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.009+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.011 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.010+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:50.011 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.010+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.011 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.010+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.011 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.010+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.013 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.010+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:50.013 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.010+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.013 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.011+0000 7f22f1cbd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.013 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.011+0000 7f22f1cbd500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:50.013 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.011+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.013 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.011+0000 7f22f1cbd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.013 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.011+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.013 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.011+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.013 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.011+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.013 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.011+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:50.014 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.012+0000 7f22f1cbd500 20 zonegroup default 2023-09-22T13:09:50.014 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.012+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.014 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.012+0000 7f22f1cbd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.014 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.012+0000 7f22f1cbd500 10 Cannot find current period zone using local zone 2023-09-22T13:09:50.014 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.012+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.014 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.012+0000 7f22f1cbd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.014 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.012+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.015 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.012+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.015 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.012+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.015 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.013+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:50.015 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.013+0000 7f22f1cbd500 20 zone default found 2023-09-22T13:09:50.015 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.013+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.015 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.013+0000 7f22f1cbd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.016 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.013+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.016 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.013+0000 7f22f1cbd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.016 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.013+0000 7f22f1cbd500 20 started sync module instance, tier type = 2023-09-22T13:09:50.016 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.013+0000 7f22f1cbd500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:50.021 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.020+0000 7f22dc99e700 0 -- 172.21.15.98:0/406226215 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f22d002f040 msgr2=0x7f22d002f420 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:50.022 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.021+0000 7f22f1cbd500 20 add_watcher() i=0 2023-09-22T13:09:50.022 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.021+0000 7f22f1cbd500 20 add_watcher() i=1 2023-09-22T13:09:50.022 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.021+0000 7f22f1cbd500 20 add_watcher() i=2 2023-09-22T13:09:50.022 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.021+0000 7f22f1cbd500 20 add_watcher() i=3 2023-09-22T13:09:50.023 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.021+0000 7f22f1cbd500 20 add_watcher() i=4 2023-09-22T13:09:50.023 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.021+0000 7f22f1cbd500 20 add_watcher() i=5 2023-09-22T13:09:50.023 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.021+0000 7f22f1cbd500 20 add_watcher() i=6 2023-09-22T13:09:50.023 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.022+0000 7f22f1cbd500 20 add_watcher() i=7 2023-09-22T13:09:50.023 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.022+0000 7f22f1cbd500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:50.025 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.024+0000 7f22ae7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:50.025 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.024+0000 7f22f1cbd500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:50.025 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.024+0000 7f22f1cbd500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:50.029 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.028+0000 7f22dd19f700 0 -- 172.21.15.98:0/2284109826 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x55cdbe657bf0 msgr2=0x55cdbe662940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:09:50.031 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.030+0000 7f22f1cbd500 5 note: GC not initialized 2023-09-22T13:09:50.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.031+0000 7f221d7fa700 20 reqs_thread_entry: start 2023-09-22T13:09:50.032 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.031+0000 7f2216ffd700 20 reqs_thread_entry: start 2023-09-22T13:09:50.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.031+0000 7f22f1cbd500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:50.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.031+0000 7f22f1cbd500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:50.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.031+0000 7f22157fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:20 2023 2023-09-22T13:09:50.033 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:50.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.031+0000 7f2214ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:50.033 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.032+0000 7f22f1cbd500 20 get_system_obj_state: rctx=0x7ffc2a07e9f0 obj=default.rgw.meta:root:check-bucket state=0x55cdbe633580 s->prefetch_data=0 2023-09-22T13:09:50.034 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.032+0000 7f22f1cbd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:50.034 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.032+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.034 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.032+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:50.034 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.032+0000 7f22f1cbd500 20 get_system_obj_state: rctx=0x7ffc2a07e9f0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55cdbe657300 s->prefetch_data=0 2023-09-22T13:09:50.035 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.033+0000 7f22f1cbd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:50.035 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.033+0000 7f22f1cbd500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:50.035 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.033+0000 7f22f1cbd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:50.035 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.033+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.035 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.033+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:50.035 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.033+0000 7f22f1cbd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:50.035 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.033+0000 7f22f1cbd500 20 get_system_obj_state: rctx=0x7ffc2a07e7c0 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55cdbe657300 s->prefetch_data=0 2023-09-22T13:09:50.035 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.034+0000 7f22f1cbd500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:50.035 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.034+0000 7f22f1cbd500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:50.035 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.034+0000 7f22f1cbd500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:50.035 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.034+0000 7f22f1cbd500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.036 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.034+0000 7f22f1cbd500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:50.036 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.034+0000 7f22f1cbd500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:50.036 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.034+0000 7f22f1cbd500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:50.036 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.036+0000 7f22f1cbd500 20 remove_watcher() i=0 2023-09-22T13:09:50.037 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.036+0000 7f22f1cbd500 2 removed watcher, disabling cache 2023-09-22T13:09:50.038 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.037+0000 7f22f1cbd500 20 remove_watcher() i=1 2023-09-22T13:09:50.038 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.038+0000 7f22f1cbd500 20 remove_watcher() i=2 2023-09-22T13:09:50.039 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.039+0000 7f22f1cbd500 20 remove_watcher() i=3 2023-09-22T13:09:50.040 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.039+0000 7f22f1cbd500 20 remove_watcher() i=4 2023-09-22T13:09:50.041 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.040+0000 7f22f1cbd500 20 remove_watcher() i=5 2023-09-22T13:09:50.042 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.041+0000 7f22f1cbd500 20 remove_watcher() i=6 2023-09-22T13:09:50.043 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.042+0000 7f22f1cbd500 20 remove_watcher() i=7 2023-09-22T13:09:50.043 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.043+0000 7f22ae7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:50.103 INFO:tasks.workunit.client.0.smithi098.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:09:50.104 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object e 2023-09-22T13:09:50.129 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:50.129 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:50.137 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.137+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.139 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.138+0000 7fb335461500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.139 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.139+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.140 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.139+0000 7fb335461500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.140 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.139+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.140 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.139+0000 7fb335461500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.140 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.139+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.141 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.140+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.353 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.352+0000 7fb335461500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:50.353 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.352+0000 7fb335461500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:50.353 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.352+0000 7fb335461500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:50.353 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.352+0000 7fb335461500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:50.353 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.352+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.354 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.352+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.354 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.352+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.354 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.353+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:50.355 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.353+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.355 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.353+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.355 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.353+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.355 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.353+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:50.355 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.353+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.355 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.354+0000 7fb335461500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.355 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.354+0000 7fb335461500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:50.355 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.354+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.355 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.354+0000 7fb335461500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.355 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.354+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.356 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.354+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.356 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.354+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.356 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.355+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:50.357 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.355+0000 7fb335461500 20 zonegroup default 2023-09-22T13:09:50.357 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.355+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.357 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.355+0000 7fb335461500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.357 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.355+0000 7fb335461500 10 Cannot find current period zone using local zone 2023-09-22T13:09:50.357 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.355+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.357 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.355+0000 7fb335461500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.357 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.355+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.357 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.356+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.357 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.356+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.356+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:50.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.356+0000 7fb335461500 20 zone default found 2023-09-22T13:09:50.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.356+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.356+0000 7fb335461500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.356+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.356+0000 7fb335461500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.359 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.356+0000 7fb335461500 20 started sync module instance, tier type = 2023-09-22T13:09:50.359 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.356+0000 7fb335461500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:50.359 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.358+0000 7fb320943700 0 -- 172.21.15.98:0/768823720 >> [v2:172.21.15.98:6808/140877,v1:172.21.15.98:6809/140877] conn(0x7fb3100030b0 msgr2=0x7fb30c01d950 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:50.365 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.364+0000 7fb335461500 20 add_watcher() i=0 2023-09-22T13:09:50.365 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.364+0000 7fb335461500 20 add_watcher() i=1 2023-09-22T13:09:50.365 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.364+0000 7fb335461500 20 add_watcher() i=2 2023-09-22T13:09:50.365 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.364+0000 7fb335461500 20 add_watcher() i=3 2023-09-22T13:09:50.365 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.364+0000 7fb335461500 20 add_watcher() i=4 2023-09-22T13:09:50.365 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.364+0000 7fb335461500 20 add_watcher() i=5 2023-09-22T13:09:50.365 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.364+0000 7fb335461500 20 add_watcher() i=6 2023-09-22T13:09:50.366 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.365+0000 7fb335461500 20 add_watcher() i=7 2023-09-22T13:09:50.366 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.365+0000 7fb335461500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:50.367 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.366+0000 7fb2f1fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:50.367 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.366+0000 7fb335461500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:50.367 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.366+0000 7fb335461500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:50.368 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.367+0000 7fb320943700 0 -- 172.21.15.98:0/2851223022 >> [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] conn(0x55e088771ca0 msgr2=0x55e08876bf50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until injecting socket failure 2023-09-22T13:09:50.372 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.372+0000 7fb335461500 5 note: GC not initialized 2023-09-22T13:09:50.374 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.373+0000 7fb268ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:50.374 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.373+0000 7fb25e7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:50.374 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.373+0000 7fb335461500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:50.374 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.373+0000 7fb335461500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:50.374 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.373+0000 7fb25cff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:20 2023 2023-09-22T13:09:50.374 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:50.374 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.373+0000 7fb243fff700 20 reqs_thread_entry: start 2023-09-22T13:09:50.375 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.373+0000 7fb335461500 20 get_system_obj_state: rctx=0x7ffc93ef2a30 obj=default.rgw.meta:root:check-bucket state=0x55e088770c10 s->prefetch_data=0 2023-09-22T13:09:50.376 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.374+0000 7fb335461500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:50.376 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.374+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.376 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.374+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:50.376 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.374+0000 7fb335461500 20 get_system_obj_state: rctx=0x7ffc93ef2a30 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55e08876f9c0 s->prefetch_data=0 2023-09-22T13:09:50.376 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.374+0000 7fb335461500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.374+0000 7fb335461500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.374+0000 7fb335461500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.374+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.375+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.375+0000 7fb335461500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.375+0000 7fb335461500 20 get_system_obj_state: rctx=0x7ffc93ef2800 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55e08876f9c0 s->prefetch_data=0 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.375+0000 7fb335461500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.375+0000 7fb335461500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.375+0000 7fb335461500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.375+0000 7fb335461500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.375+0000 7fb335461500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:50.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.375+0000 7fb335461500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:50.378 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.375+0000 7fb335461500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:50.378 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.377+0000 7fb335461500 20 remove_watcher() i=0 2023-09-22T13:09:50.378 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.377+0000 7fb335461500 2 removed watcher, disabling cache 2023-09-22T13:09:50.378 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.378+0000 7fb335461500 20 remove_watcher() i=1 2023-09-22T13:09:50.379 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.379+0000 7fb335461500 20 remove_watcher() i=2 2023-09-22T13:09:50.380 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.380+0000 7fb335461500 20 remove_watcher() i=3 2023-09-22T13:09:50.381 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.380+0000 7fb335461500 20 remove_watcher() i=4 2023-09-22T13:09:50.382 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.381+0000 7fb335461500 20 remove_watcher() i=5 2023-09-22T13:09:50.383 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.382+0000 7fb335461500 20 remove_watcher() i=6 2023-09-22T13:09:50.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.383+0000 7fb335461500 20 remove_watcher() i=7 2023-09-22T13:09:50.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.384+0000 7fb2f1fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:50.441 INFO:tasks.workunit.client.0.smithi098.stderr:An error occurred (NoSuchKey) when calling the PutObject operation: None 2023-09-22T13:09:50.442 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bi list --bucket check-bucket --object f 2023-09-22T13:09:50.466 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:50.466 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:50.474 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.473+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.476 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.475+0000 7fde05d72500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.476 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.475+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.476 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.476+0000 7fde05d72500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.476 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.476+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.477 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.476+0000 7fde05d72500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.477 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.476+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.477 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.477+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.488 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.486+0000 7fde05d72500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:50.488 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.486+0000 7fde05d72500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:50.488 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.486+0000 7fde05d72500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:50.488 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.486+0000 7fde05d72500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:50.488 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.486+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.488 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.487+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.488 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.487+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.488 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.487+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:50.488 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.487+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.488 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.487+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.489 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.487+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.490 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.487+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:50.491 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.487+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.491 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.488+0000 7fde05d72500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.491 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.488+0000 7fde05d72500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:50.491 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.488+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.491 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.488+0000 7fde05d72500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.491 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.488+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.491 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.488+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.491 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.488+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.491 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.489+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:50.491 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.489+0000 7fde05d72500 20 zonegroup default 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.489+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.489+0000 7fde05d72500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.489+0000 7fde05d72500 10 Cannot find current period zone using local zone 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.489+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.489+0000 7fde05d72500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.489+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.489+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.489+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.490+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.490+0000 7fde05d72500 20 zone default found 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.490+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.490+0000 7fde05d72500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.493 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.490+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.493 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.490+0000 7fde05d72500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:50.493 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.490+0000 7fde05d72500 20 started sync module instance, tier type = 2023-09-22T13:09:50.493 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.490+0000 7fde05d72500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:50.495 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.494+0000 7fddf1a55700 0 -- 172.21.15.98:0/1035571001 >> [v2:172.21.15.169:6821/140771,v1:172.21.15.169:6823/140771] conn(0x7fdde400e8a0 msgr2=0x7fdde401d360 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:50.498 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.497+0000 7fde05d72500 20 add_watcher() i=0 2023-09-22T13:09:50.498 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.497+0000 7fde05d72500 20 add_watcher() i=1 2023-09-22T13:09:50.498 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.497+0000 7fde05d72500 20 add_watcher() i=2 2023-09-22T13:09:50.498 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.497+0000 7fde05d72500 20 add_watcher() i=3 2023-09-22T13:09:50.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.497+0000 7fde05d72500 20 add_watcher() i=4 2023-09-22T13:09:50.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.497+0000 7fde05d72500 20 add_watcher() i=5 2023-09-22T13:09:50.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.497+0000 7fde05d72500 20 add_watcher() i=6 2023-09-22T13:09:50.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.498+0000 7fde05d72500 20 add_watcher() i=7 2023-09-22T13:09:50.499 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.498+0000 7fde05d72500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:50.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.499+0000 7fddc27e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:50.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.499+0000 7fde05d72500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:50.500 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.499+0000 7fde05d72500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:50.502 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.501+0000 7fddf0a53700 0 -- 172.21.15.98:0/762845107 >> [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] conn(0x55c576e19200 msgr2=0x55c576e19680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:50.506 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.505+0000 7fde05d72500 5 note: GC not initialized 2023-09-22T13:09:50.508 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.506+0000 7fdd317fa700 20 reqs_thread_entry: start 2023-09-22T13:09:50.508 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.506+0000 7fdd2affd700 20 reqs_thread_entry: start 2023-09-22T13:09:50.508 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.506+0000 7fde05d72500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:50.508 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.507+0000 7fde05d72500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:50.508 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.507+0000 7fdd297fa700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:20 2023 2023-09-22T13:09:50.508 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:50.508 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.507+0000 7fdd28ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:50.508 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.507+0000 7fde05d72500 20 get_system_obj_state: rctx=0x7fff6de69f30 obj=default.rgw.meta:root:check-bucket state=0x55c576c6c810 s->prefetch_data=0 2023-09-22T13:09:50.510 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.507+0000 7fde05d72500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:50.510 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.507+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.510 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.507+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:50.510 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.507+0000 7fde05d72500 20 get_system_obj_state: rctx=0x7fff6de69f30 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55c576c26910 s->prefetch_data=0 2023-09-22T13:09:50.510 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.508+0000 7fde05d72500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:50.510 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.508+0000 7fde05d72500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:50.510 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.508+0000 7fde05d72500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:50.510 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.508+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.510 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.508+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:50.511 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.508+0000 7fde05d72500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:50.511 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.508+0000 7fde05d72500 20 get_system_obj_state: rctx=0x7fff6de69d00 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55c576c26910 s->prefetch_data=0 2023-09-22T13:09:50.511 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.508+0000 7fde05d72500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:50.511 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.508+0000 7fde05d72500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:50.511 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.508+0000 7fde05d72500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:50.511 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.508+0000 7fde05d72500 20 rados->read ofs=0 len=0 2023-09-22T13:09:50.511 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.509+0000 7fde05d72500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:50.511 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.509+0000 7fde05d72500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:50.511 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.509+0000 7fde05d72500 20 bucket index oid: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:50.512 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.511+0000 7fde05d72500 20 remove_watcher() i=0 2023-09-22T13:09:50.512 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.511+0000 7fde05d72500 2 removed watcher, disabling cache 2023-09-22T13:09:50.512 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.511+0000 7fde05d72500 20 remove_watcher() i=1 2023-09-22T13:09:50.513 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.512+0000 7fde05d72500 20 remove_watcher() i=2 2023-09-22T13:09:50.514 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.513+0000 7fde05d72500 20 remove_watcher() i=3 2023-09-22T13:09:50.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.514+0000 7fde05d72500 20 remove_watcher() i=4 2023-09-22T13:09:50.515 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.515+0000 7fde05d72500 20 remove_watcher() i=5 2023-09-22T13:09:50.516 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.515+0000 7fde05d72500 20 remove_watcher() i=6 2023-09-22T13:09:50.517 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.516+0000 7fde05d72500 20 remove_watcher() i=7 2023-09-22T13:09:50.518 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:50.517+0000 7fddc27e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:50.528 INFO:tasks.workunit.client.0.smithi098.stderr:ceph config rm client rgw_debug_inject_set_olh_err 2023-09-22T13:09:50.918 INFO:tasks.workunit.client.0.smithi098.stderr:ceph config rm client rgw_debug_inject_olh_cancel_modification_err 2023-09-22T13:09:51.311 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bucket check --fix --bucket check-bucket 2023-09-22T13:09:51.334 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:51.334 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:51.342 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.342+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.345 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.344+0000 7fcf10fc2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.345 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.344+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.345 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.345+0000 7fcf10fc2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.346 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.345+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.346 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.345+0000 7fcf10fc2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.346 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.345+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.346 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.345+0000 7fcf10fc2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.357 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.356+0000 7fcf10fc2500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:51.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.356+0000 7fcf10fc2500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:51.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.356+0000 7fcf10fc2500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:51.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.356+0000 7fcf10fc2500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:51.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.356+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.356+0000 7fcf10fc2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.356+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.357+0000 7fcf10fc2500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:51.358 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.357+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.360 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.357+0000 7fcf10fc2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.360 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.357+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.357+0000 7fcf10fc2500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.357+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.357+0000 7fcf10fc2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.357+0000 7fcf10fc2500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.357+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.358+0000 7fcf10fc2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.358+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.358+0000 7fcf10fc2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.358+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.358+0000 7fcf10fc2500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.358+0000 7fcf10fc2500 20 zonegroup default 2023-09-22T13:09:51.361 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.358+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.359+0000 7fcf10fc2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.359+0000 7fcf10fc2500 10 Cannot find current period zone using local zone 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.359+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.359+0000 7fcf10fc2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.359+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.359+0000 7fcf10fc2500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.359+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.359+0000 7fcf10fc2500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.359+0000 7fcf10fc2500 20 zone default found 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.360+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.360+0000 7fcf10fc2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.362 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.360+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.363 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.360+0000 7fcf10fc2500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.363 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.360+0000 7fcf10fc2500 20 started sync module instance, tier type = 2023-09-22T13:09:51.363 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.360+0000 7fcf10fc2500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:51.368 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.367+0000 7fcf10fc2500 20 add_watcher() i=0 2023-09-22T13:09:51.368 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.367+0000 7fcf10fc2500 20 add_watcher() i=1 2023-09-22T13:09:51.368 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.367+0000 7fcf10fc2500 20 add_watcher() i=2 2023-09-22T13:09:51.369 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.367+0000 7fcf10fc2500 20 add_watcher() i=3 2023-09-22T13:09:51.369 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.367+0000 7fcf10fc2500 20 add_watcher() i=4 2023-09-22T13:09:51.369 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.367+0000 7fcf10fc2500 20 add_watcher() i=5 2023-09-22T13:09:51.369 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.367+0000 7fcf10fc2500 20 add_watcher() i=6 2023-09-22T13:09:51.369 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.368+0000 7fcf10fc2500 20 add_watcher() i=7 2023-09-22T13:09:51.369 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.368+0000 7fcf10fc2500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:51.371 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.370+0000 7fcefcca5700 0 -- 172.21.15.98:0/3698036320 >> [v2:172.21.15.169:6821/140771,v1:172.21.15.169:6823/140771] conn(0x7fceec04c470 msgr2=0x7fceec04b520 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:51.371 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.370+0000 7fcecdfe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:51.371 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.371+0000 7fcf10fc2500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:51.371 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.371+0000 7fcf10fc2500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:51.375 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.374+0000 7fcefbca3700 0 -- 172.21.15.98:0/3186274477 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x555d01a518c0 msgr2=0x555d01a56ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:51.377 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.376+0000 7fcf10fc2500 5 note: GC not initialized 2023-09-22T13:09:51.379 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.377+0000 7fce44ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:51.379 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fce3a7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:51.379 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fcf10fc2500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:51.379 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fcf10fc2500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:51.379 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fce38ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:21 2023 2023-09-22T13:09:51.379 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:51.379 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fce1ffff700 20 reqs_thread_entry: start 2023-09-22T13:09:51.380 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fcf10fc2500 20 get_system_obj_state: rctx=0x7ffcc966c898 obj=default.rgw.meta:root:check-bucket state=0x555d01a1a250 s->prefetch_data=0 2023-09-22T13:09:51.380 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fcf10fc2500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:09:51.381 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fcf10fc2500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:09:51.382 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fcf10fc2500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:51.382 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fcf10fc2500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:51.382 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fcf10fc2500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:51.382 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fcf10fc2500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:09:51.382 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.378+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.382 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:51.382 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:09:51.382 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:51.382 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 20 get_system_obj_state: rctx=0x7ffcc966c968 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x555d01a1a250 s->prefetch_data=0 2023-09-22T13:09:51.383 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : miss 2023-09-22T13:09:51.383 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x16 2023-09-22T13:09:51.383 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:51.383 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:51.383 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:51.383 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:51.383 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:51.383 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:51.383 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:09:51.383 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.379+0000 7fcf10fc2500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x13 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 20 RGWRados::Bucket::List::list_objects_ordered starting attempt 1 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 10 RGWRados::cls_bucket_list_ordered: :check-bucket[8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1]) start_after="_multipart_[]", prefix="_multipart_" num_entries=1001, list_versions=1, expansion_factor=1 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 10 RGWRados::cls_bucket_list_ordered request from each of 1 shard(s) for 1001 entries to get 1001 total entries 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 20 RGWRados::cls_bucket_list_ordered: returning, count=0, is_truncated=0 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 20 RGWRados::cls_bucket_list_ordered: returning, last_entry NOT SET 2023-09-22T13:09:51.384 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.380+0000 7fcf10fc2500 20 RGWRados::Bucket::List::list_objects_ordered INFO end of outer loop, truncated=0, count=0, attempt=1 2023-09-22T13:09:51.385 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.384+0000 7fcf10fc2500 20 remove_watcher() i=0 2023-09-22T13:09:51.385 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.384+0000 7fcf10fc2500 2 removed watcher, disabling cache 2023-09-22T13:09:51.385 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.385+0000 7fcf10fc2500 20 remove_watcher() i=1 2023-09-22T13:09:51.386 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.386+0000 7fcf10fc2500 20 remove_watcher() i=2 2023-09-22T13:09:51.387 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.387+0000 7fcf10fc2500 20 remove_watcher() i=3 2023-09-22T13:09:51.388 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.388+0000 7fcf10fc2500 20 remove_watcher() i=4 2023-09-22T13:09:51.389 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.388+0000 7fcf10fc2500 20 remove_watcher() i=5 2023-09-22T13:09:51.390 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.389+0000 7fcf10fc2500 20 remove_watcher() i=6 2023-09-22T13:09:51.391 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.390+0000 7fcf10fc2500 20 remove_watcher() i=7 2023-09-22T13:09:51.391 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.391+0000 7fcecdfe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:51.397 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bucket check unlinked --bucket check-bucket --fix --min-age-hours 0 --rgw-olh-pending-timeout-sec 0 --dump-keys 2023-09-22T13:09:51.425 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:51.425 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:51.433 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.433+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.435 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.434+0000 7ffaf4b56500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.435 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.434+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.435+0000 7ffaf4b56500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.435+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.435+0000 7ffaf4b56500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.436 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.435+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.437 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.436+0000 7ffaf4b56500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.649 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.648+0000 7ffaf4b56500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:51.649 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.648+0000 7ffaf4b56500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:51.649 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.648+0000 7ffaf4b56500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:51.649 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.648+0000 7ffaf4b56500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:51.650 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.648+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.650 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.648+0000 7ffaf4b56500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.650 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.648+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.650 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.648+0000 7ffaf4b56500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:51.650 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.648+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.650 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.648+0000 7ffaf4b56500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.650 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.648+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.651 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.649+0000 7ffaf4b56500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:51.652 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.649+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.652 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.649+0000 7ffaf4b56500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.652 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.649+0000 7ffaf4b56500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:51.652 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.649+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.652 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.649+0000 7ffaf4b56500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.652 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.649+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.652 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.650+0000 7ffaf4b56500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.652 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.650+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.652 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.650+0000 7ffaf4b56500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:51.652 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.650+0000 7ffaf4b56500 20 zonegroup default 2023-09-22T13:09:51.652 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.650+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.653 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.650+0000 7ffaf4b56500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.653 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.650+0000 7ffaf4b56500 10 Cannot find current period zone using local zone 2023-09-22T13:09:51.653 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.650+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.653 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.651+0000 7ffaf4b56500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.653 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.651+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.654 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.651+0000 7ffaf4b56500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.654 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.651+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.654 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.651+0000 7ffaf4b56500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:51.654 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.651+0000 7ffaf4b56500 20 zone default found 2023-09-22T13:09:51.654 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.651+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.654 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.652+0000 7ffaf4b56500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.654 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.652+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.654 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.652+0000 7ffaf4b56500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.654 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.652+0000 7ffaf4b56500 20 started sync module instance, tier type = 2023-09-22T13:09:51.655 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.652+0000 7ffaf4b56500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:51.655 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.654+0000 7ffae0038700 0 -- 172.21.15.98:0/2633882443 >> [v2:172.21.15.98:6808/140877,v1:172.21.15.98:6809/140877] conn(0x7ffad00502e0 msgr2=0x7ffad004f450 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:51.660 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.659+0000 7ffaf4b56500 20 add_watcher() i=0 2023-09-22T13:09:51.660 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.659+0000 7ffaf4b56500 20 add_watcher() i=1 2023-09-22T13:09:51.660 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.659+0000 7ffaf4b56500 20 add_watcher() i=2 2023-09-22T13:09:51.660 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.659+0000 7ffaf4b56500 20 add_watcher() i=3 2023-09-22T13:09:51.661 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.659+0000 7ffaf4b56500 20 add_watcher() i=4 2023-09-22T13:09:51.661 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.659+0000 7ffaf4b56500 20 add_watcher() i=5 2023-09-22T13:09:51.661 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.659+0000 7ffaf4b56500 20 add_watcher() i=6 2023-09-22T13:09:51.661 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.660+0000 7ffaf4b56500 20 add_watcher() i=7 2023-09-22T13:09:51.661 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.660+0000 7ffaf4b56500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:51.663 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.662+0000 7ffab1fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:51.663 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.662+0000 7ffaf4b56500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:51.663 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.662+0000 7ffaf4b56500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:51.664 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.664+0000 7ffae0038700 0 -- 172.21.15.98:0/3077825766 >> [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] conn(0x55e089860d00 msgr2=0x55e089858470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0)._try_send injecting socket failure 2023-09-22T13:09:51.669 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.668+0000 7ffaf4b56500 5 note: GC not initialized 2023-09-22T13:09:51.670 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.669+0000 7ffa28ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:51.670 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.669+0000 7ffa1e7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:51.670 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.670+0000 7ffaf4b56500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:51.671 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.670+0000 7ffaf4b56500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:51.672 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.670+0000 7ffa1cff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:22 2023 2023-09-22T13:09:51.672 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:51.672 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.670+0000 7ffa03fff700 20 reqs_thread_entry: start 2023-09-22T13:09:51.672 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.670+0000 7ffaf4b56500 20 get_system_obj_state: rctx=0x7fff8dd90828 obj=default.rgw.meta:root:check-bucket state=0x55e0897c8810 s->prefetch_data=0 2023-09-22T13:09:51.672 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.670+0000 7ffaf4b56500 10 cache get: name=default.rgw.meta+root+check-bucket : miss 2023-09-22T13:09:51.672 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x16 2023-09-22T13:09:51.672 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 adding default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:51.672 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:51.672 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:51.672 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 cache get: name=default.rgw.meta+root+check-bucket : type miss (requested=0x11, cached=0x16) 2023-09-22T13:09:51.672 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:51.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 cache put: name=default.rgw.meta+root+check-bucket info.flags=0x11 2023-09-22T13:09:51.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 moving default.rgw.meta+root+check-bucket to cache LRU end 2023-09-22T13:09:51.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 20 get_system_obj_state: rctx=0x7fff8dd908f8 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55e0897c8810 s->prefetch_data=0 2023-09-22T13:09:51.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : miss 2023-09-22T13:09:51.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x16 2023-09-22T13:09:51.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 adding default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:51.674 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.671+0000 7ffaf4b56500 10 cache get: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 : type miss (requested=0x13, cached=0x16) 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.672+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.672+0000 7ffaf4b56500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.672+0000 7ffaf4b56500 10 cache put: name=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 info.flags=0x13 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.672+0000 7ffaf4b56500 10 moving default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 to cache LRU end 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.672+0000 7ffaf4b56500 10 updating xattr: name=ceph.objclass.version bl.length()=42 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.672+0000 7ffaf4b56500 10 updating xattr: name=user.rgw.acl bl.length()=179 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.672+0000 7ffaf4b56500 10 chain_cache_entry: cache_locator=default.rgw.meta+root+.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 2023-09-22T13:09:51.675 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.672+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.676 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.673+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.676 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.673+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:c state=0x55e0898fff68 s->prefetch_data=0 2023-09-22T13:09:51.676 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.674+0000 7ffaf4b56500 20 get_obj_state: setting s->obj_tag to wsrolnd8a1azixb3x221wki9s7xgdq4f 2023-09-22T13:09:51.676 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.674+0000 7ffaf4b56500 20 get_obj_state_impl: setting s->olh_tag to fs5ycgk2a3jxhay2b3wy5z3tjw07xyta 2023-09-22T13:09:51.676 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.674+0000 7ffaf4b56500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=fs5ycgk2a3jxhay2b3wy5z3tjw07xyta 2023-09-22T13:09:51.676 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.676+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.677 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.677+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.677 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.677+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.678 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.677+0000 7ffaf4b56500 20 olh_log_entry: epoch=3 op=3 key=c[yEkcXzTUJSJhzQD5-Gy3k7nYYGt6y.4] 2023-09-22T13:09:51.678 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.677+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:_:yEkcXzTUJSJhzQD5-Gy3k7nYYGt6y.4_c state=0x55e089909708 s->prefetch_data=0 2023-09-22T13:09:51.679 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.678+0000 7ffaf4b56500 10 manifest: total_size = 12 2023-09-22T13:09:51.679 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.678+0000 7ffaf4b56500 20 get_obj_state: setting s->obj_tag to 8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4430.6795737266428862009 2023-09-22T13:09:51.679 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.678+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:_:yEkcXzTUJSJhzQD5-Gy3k7nYYGt6y.4_c state=0x55e089909708 s->prefetch_data=0 2023-09-22T13:09:51.679 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.678+0000 7ffaf4b56500 20 prepare_atomic_modification: state is not atomic. state=0x55e089909708 2023-09-22T13:09:51.679 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.678+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.681 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.680+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : miss 2023-09-22T13:09:51.681 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.680+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.681 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.680+0000 7ffaf4b56500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.681 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.680+0000 7ffaf4b56500 10 cache put: name=default.rgw.log++bucket.sync-source-hints.check-bucket info.flags=0x0 2023-09-22T13:09:51.681 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.680+0000 7ffaf4b56500 10 adding default.rgw.log++bucket.sync-source-hints.check-bucket to cache LRU end 2023-09-22T13:09:51.681 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.680+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : miss 2023-09-22T13:09:51.681 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.680+0000 7ffaf4b56500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.682 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.681+0000 7ffaf4b56500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.682 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.681+0000 7ffaf4b56500 10 cache put: name=default.rgw.log++bucket.sync-target-hints.check-bucket info.flags=0x0 2023-09-22T13:09:51.682 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.681+0000 7ffaf4b56500 10 adding default.rgw.log++bucket.sync-target-hints.check-bucket to cache LRU end 2023-09-22T13:09:51.682 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.681+0000 7ffaf4b56500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:51.682 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.681+0000 7ffaf4b56500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:51.682 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.681+0000 7ffaf4b56500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:51.683 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.682+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.683 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.682+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.684 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.683+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.684 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.683+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.684 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.683+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.684 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.683+0000 7ffaf4b56500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:51.684 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.683+0000 7ffaf4b56500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:51.684 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.683+0000 7ffaf4b56500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:51.684 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.683+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.685 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.683+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:d state=0x55e0898fff68 s->prefetch_data=0 2023-09-22T13:09:51.685 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.684+0000 7ffaf4b56500 20 get_obj_state: setting s->obj_tag to tqzjcqmrfwyqubylhgr43kmix3ovyje9 2023-09-22T13:09:51.685 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.684+0000 7ffaf4b56500 20 get_obj_state_impl: setting s->olh_tag to k1pdfn0o8e30ddi7bmbo69y7rie4nnxs 2023-09-22T13:09:51.685 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.684+0000 7ffaf4b56500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=k1pdfn0o8e30ddi7bmbo69y7rie4nnxs 2023-09-22T13:09:51.686 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.685+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.687 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.686+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.687 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.686+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.687 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.686+0000 7ffaf4b56500 20 olh_log_entry: epoch=3 op=3 key=d[wX3VmdgEPCOXxygUl2USZao7dWuMOyd] 2023-09-22T13:09:51.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.686+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:_:wX3VmdgEPCOXxygUl2USZao7dWuMOyd_d state=0x55e0899086f8 s->prefetch_data=0 2023-09-22T13:09:51.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.687+0000 7ffaf4b56500 10 manifest: total_size = 11 2023-09-22T13:09:51.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.687+0000 7ffaf4b56500 20 get_obj_state: setting s->obj_tag to 8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4430.16432177197187826667 2023-09-22T13:09:51.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.687+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:_:wX3VmdgEPCOXxygUl2USZao7dWuMOyd_d state=0x55e0899086f8 s->prefetch_data=0 2023-09-22T13:09:51.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.687+0000 7ffaf4b56500 20 prepare_atomic_modification: state is not atomic. state=0x55e0899086f8 2023-09-22T13:09:51.688 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.687+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.690 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.689+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.690 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.689+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.690 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.689+0000 7ffaf4b56500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:51.690 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.689+0000 7ffaf4b56500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:51.690 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.689+0000 7ffaf4b56500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:51.691 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.690+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.691 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.690+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.691+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.691+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.691+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.691+0000 7ffaf4b56500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:51.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.691+0000 7ffaf4b56500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:51.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.691+0000 7ffaf4b56500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:51.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.691+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.692+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:e state=0x55e0898fff68 s->prefetch_data=0 2023-09-22T13:09:51.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.692+0000 7ffaf4b56500 20 get_obj_state: setting s->obj_tag to rp694bp5soqvcqoqs9ok0lt8awqk1ph4 2023-09-22T13:09:51.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.692+0000 7ffaf4b56500 20 get_obj_state_impl: setting s->olh_tag to fikffyvvc634mjim6gnvy7masjs7r6k1 2023-09-22T13:09:51.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.692+0000 7ffaf4b56500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=fikffyvvc634mjim6gnvy7masjs7r6k1 2023-09-22T13:09:51.694 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.693+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.695 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.695+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.695 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.695+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.695+0000 7ffaf4b56500 20 olh_log_entry: epoch=2 op=2 key=e[] 2023-09-22T13:09:51.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.695+0000 7ffaf4b56500 20 olh_log_entry: epoch=2 op=3 key=e[Njm00b0UkbuJe7kU7YEtknDa23GYhlK] 2023-09-22T13:09:51.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.695+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:_:Njm00b0UkbuJe7kU7YEtknDa23GYhlK_e state=0x55e089909af8 s->prefetch_data=0 2023-09-22T13:09:51.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.695+0000 7ffaf4b56500 10 manifest: total_size = 9 2023-09-22T13:09:51.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.695+0000 7ffaf4b56500 20 get_obj_state: setting s->obj_tag to 8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4430.8612205923526306571 2023-09-22T13:09:51.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.695+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:_:Njm00b0UkbuJe7kU7YEtknDa23GYhlK_e state=0x55e089909af8 s->prefetch_data=0 2023-09-22T13:09:51.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.695+0000 7ffaf4b56500 20 prepare_atomic_modification: state is not atomic. state=0x55e089909af8 2023-09-22T13:09:51.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.695+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.698 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.698+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.698+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.698+0000 7ffaf4b56500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:51.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.698+0000 7ffaf4b56500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:51.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.698+0000 7ffaf4b56500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:51.699 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.699+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:e state=0x55e0898fff68 s->prefetch_data=0 2023-09-22T13:09:51.700 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.699+0000 7ffaf4b56500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=fikffyvvc634mjim6gnvy7masjs7r6k1 2023-09-22T13:09:51.702 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.701+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.703 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.702+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.703 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.702+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.704 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.702+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.704 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.702+0000 7ffaf4b56500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:51.704 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.702+0000 7ffaf4b56500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:51.704 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.702+0000 7ffaf4b56500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:51.704 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.703+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.704 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.703+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:f state=0x55e0898fff68 s->prefetch_data=0 2023-09-22T13:09:51.705 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.703+0000 7ffaf4b56500 20 get_obj_state: setting s->obj_tag to u0b34g3d66l5ej4hpxayx15n7xgjj2qg 2023-09-22T13:09:51.705 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.703+0000 7ffaf4b56500 20 get_obj_state_impl: setting s->olh_tag to h4n9e17n6jusex757ke3l09rpfrf4jue 2023-09-22T13:09:51.705 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.703+0000 7ffaf4b56500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=h4n9e17n6jusex757ke3l09rpfrf4jue 2023-09-22T13:09:51.705 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.704+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.707 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.706+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.707 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.706+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.707 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.706+0000 7ffaf4b56500 20 olh_log_entry: epoch=2 op=2 key=f[] 2023-09-22T13:09:51.707 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.706+0000 7ffaf4b56500 20 olh_log_entry: epoch=2 op=3 key=f[Hxsk5Ye6-HqFNzNx8gF0FH5z5cLlvos] 2023-09-22T13:09:51.707 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.706+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:_:Hxsk5Ye6-HqFNzNx8gF0FH5z5cLlvos_f state=0x55e0899086f8 s->prefetch_data=0 2023-09-22T13:09:51.707 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.706+0000 7ffaf4b56500 10 manifest: total_size = 11 2023-09-22T13:09:51.708 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.706+0000 7ffaf4b56500 20 get_obj_state: setting s->obj_tag to 8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4430.512627728616596474 2023-09-22T13:09:51.708 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.706+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:_:Hxsk5Ye6-HqFNzNx8gF0FH5z5cLlvos_f state=0x55e0899086f8 s->prefetch_data=0 2023-09-22T13:09:51.708 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.706+0000 7ffaf4b56500 20 prepare_atomic_modification: state is not atomic. state=0x55e0899086f8 2023-09-22T13:09:51.708 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.706+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.709 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.708+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.709 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.708+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.709 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.708+0000 7ffaf4b56500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:51.709 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.708+0000 7ffaf4b56500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:51.709 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.708+0000 7ffaf4b56500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:51.710 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.709+0000 7ffaf4b56500 20 get_obj_state: rctx=0x55e0898fed70 obj=check-bucket:f state=0x55e0898fff68 s->prefetch_data=0 2023-09-22T13:09:51.710 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.709+0000 7ffaf4b56500 20 bucket_index_guard_olh_op(): olh_state.olh_tag=h4n9e17n6jusex757ke3l09rpfrf4jue 2023-09-22T13:09:51.712 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.712+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.714 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.713+0000 7ffaf4b56500 20 bucket index object: default.rgw.buckets.index:.dir.8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1.0 2023-09-22T13:09:51.714 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.713+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-source-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.714 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.713+0000 7ffaf4b56500 10 cache get: name=default.rgw.log++bucket.sync-target-hints.check-bucket : hit (negative entry) 2023-09-22T13:09:51.714 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.713+0000 7ffaf4b56500 10 chain_cache_entry: cache_locator= 2023-09-22T13:09:51.714 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.713+0000 7ffaf4b56500 20 chain_cache_entry: couldn't find cache locator 2023-09-22T13:09:51.714 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.713+0000 7ffaf4b56500 20 couldn't put bucket_sync_policy cache entry, might have raced with data changes 2023-09-22T13:09:51.714 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.713+0000 7ffaf4b56500 1 NOTICE: finished shard 0 (4 entries removed) 2023-09-22T13:09:51.714 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.713+0000 7ffaf4b56500 1 NOTICE: finished all shards (4 entries removed) 2023-09-22T13:09:51.715 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.714+0000 7ffaf4b56500 20 remove_watcher() i=0 2023-09-22T13:09:51.715 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.714+0000 7ffaf4b56500 2 removed watcher, disabling cache 2023-09-22T13:09:51.715 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.715+0000 7ffaf4b56500 20 remove_watcher() i=1 2023-09-22T13:09:51.716 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.715+0000 7ffaf4b56500 20 remove_watcher() i=2 2023-09-22T13:09:51.717 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.716+0000 7ffaf4b56500 20 remove_watcher() i=3 2023-09-22T13:09:51.718 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.717+0000 7ffaf4b56500 20 remove_watcher() i=4 2023-09-22T13:09:51.718 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.718+0000 7ffaf4b56500 20 remove_watcher() i=5 2023-09-22T13:09:51.719 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.719+0000 7ffaf4b56500 20 remove_watcher() i=6 2023-09-22T13:09:51.721 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.720+0000 7ffaf4b56500 20 remove_watcher() i=7 2023-09-22T13:09:51.721 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.721+0000 7ffab1fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:51.783 INFO:tasks.workunit.client.0.smithi098.stderr:radosgw-admin bucket stats --bucket check-bucket 2023-09-22T13:09:51.805 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:09:51.805 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:09:51.813 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.812+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.815 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.814+0000 7fe75f539500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.815 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.814+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.816 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.815+0000 7fe75f539500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.816 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.815+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.816 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.815+0000 7fe75f539500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.816 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.815+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.816 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.815+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.826 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.825+0000 7fe75f539500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:09:51.826 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.825+0000 7fe75f539500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:09:51.826 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.825+0000 7fe75f539500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:09:51.826 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.825+0000 7fe75f539500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:09:51.826 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.825+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.826 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.825+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.827 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.825+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.827 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.825+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:51.827 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.825+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.827 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.825+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.827 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.825+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.829 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.826+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:51.829 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.826+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.829 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.826+0000 7fe75f539500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.829 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.826+0000 7fe75f539500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:09:51.829 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.826+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.829 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.826+0000 7fe75f539500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.829 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.826+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.829 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.826+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.830 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.826+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.830 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.827+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:09:51.830 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.827+0000 7fe75f539500 20 zonegroup default 2023-09-22T13:09:51.830 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.827+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.830 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.827+0000 7fe75f539500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.830 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.827+0000 7fe75f539500 10 Cannot find current period zone using local zone 2023-09-22T13:09:51.830 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.827+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.830 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.827+0000 7fe75f539500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.830 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.827+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.830 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.827+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:09:51.830 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.827+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.831 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.827+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:09:51.831 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.828+0000 7fe75f539500 20 zone default found 2023-09-22T13:09:51.831 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.828+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.831 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.828+0000 7fe75f539500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.831 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.828+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.831 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.828+0000 7fe75f539500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:09:51.831 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.828+0000 7fe75f539500 20 started sync module instance, tier type = 2023-09-22T13:09:51.831 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.828+0000 7fe75f539500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:09:51.834 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.834+0000 7fe74a21a700 0 -- 172.21.15.98:0/2823810236 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7fe74004dfc0 msgr2=0x7fe74001f6a0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:09:51.836 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.835+0000 7fe75f539500 20 add_watcher() i=0 2023-09-22T13:09:51.836 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.835+0000 7fe75f539500 20 add_watcher() i=1 2023-09-22T13:09:51.836 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.835+0000 7fe75f539500 20 add_watcher() i=2 2023-09-22T13:09:51.836 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.835+0000 7fe75f539500 20 add_watcher() i=3 2023-09-22T13:09:51.836 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.835+0000 7fe75f539500 20 add_watcher() i=4 2023-09-22T13:09:51.836 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.835+0000 7fe75f539500 20 add_watcher() i=5 2023-09-22T13:09:51.836 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.835+0000 7fe75f539500 20 add_watcher() i=6 2023-09-22T13:09:51.837 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.836+0000 7fe75f539500 20 add_watcher() i=7 2023-09-22T13:09:51.837 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.836+0000 7fe75f539500 2 all 8 watchers are set, enabling cache 2023-09-22T13:09:51.838 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.837+0000 7fe71bfe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:51.838 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.837+0000 7fe75f539500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:09:51.838 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.837+0000 7fe75f539500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:09:51.841 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.841+0000 7fe74aa1b700 0 -- 172.21.15.98:0/452993922 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x55f31d5af120 msgr2=0x55f31d5cad10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:09:51.843 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.843+0000 7fe75f539500 5 note: GC not initialized 2023-09-22T13:09:51.845 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.844+0000 7fe68affd700 20 reqs_thread_entry: start 2023-09-22T13:09:51.845 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.844+0000 7fe688ff9700 20 reqs_thread_entry: start 2023-09-22T13:09:51.845 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.844+0000 7fe75f539500 20 init_complete bucket index max shards: 1 2023-09-22T13:09:51.845 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.844+0000 7fe75f539500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:09:51.845 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.844+0000 7fe67affd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:10:22 2023 2023-09-22T13:09:51.845 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:09:51.845 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.844+0000 7fe67a7fc700 20 reqs_thread_entry: start 2023-09-22T13:09:51.845 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.844+0000 7fe75f539500 20 get_system_obj_state: rctx=0x7fff37cab438 obj=default.rgw.meta:root:check-bucket state=0x55f31d5258f0 s->prefetch_data=0 2023-09-22T13:09:51.847 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.845+0000 7fe75f539500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:51.847 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.845+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.847 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.845+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:51.847 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.845+0000 7fe75f539500 20 get_system_obj_state: rctx=0x7fff37cab508 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55f31d5258f0 s->prefetch_data=0 2023-09-22T13:09:51.847 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.845+0000 7fe75f539500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:51.847 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.845+0000 7fe75f539500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:51.847 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.845+0000 7fe75f539500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:51.847 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.845+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.847 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.846+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:51.847 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.846+0000 7fe75f539500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:51.847 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.846+0000 7fe75f539500 20 get_system_obj_state: rctx=0x7fff37cab800 obj=default.rgw.meta:root:check-bucket state=0x55f31d5258f0 s->prefetch_data=0 2023-09-22T13:09:51.848 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.846+0000 7fe75f539500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:51.848 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.846+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.849 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.847+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=190 2023-09-22T13:09:51.849 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.847+0000 7fe75f539500 20 get_system_obj_state: rctx=0x7fff37cab800 obj=default.rgw.meta:root:.bucket.meta.check-bucket:8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.1 state=0x55f31d4a7970 s->prefetch_data=0 2023-09-22T13:09:51.849 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.847+0000 7fe75f539500 20 get_system_obj_state: s->obj_tag was set empty 2023-09-22T13:09:51.849 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.847+0000 7fe75f539500 20 Read xattr: user.rgw.acl 2023-09-22T13:09:51.849 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.847+0000 7fe75f539500 20 Read xattr: user.rgw.idtag 2023-09-22T13:09:51.849 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.847+0000 7fe75f539500 20 rados->read ofs=0 len=0 2023-09-22T13:09:51.849 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.847+0000 7fe75f539500 20 rados_obj.operate() r=0 bl.length=368 2023-09-22T13:09:51.849 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.847+0000 7fe75f539500 20 couldn't put binfo cache entry, might have raced with data changes 2023-09-22T13:09:51.849 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.849+0000 7fe75f539500 20 remove_watcher() i=0 2023-09-22T13:09:51.850 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.849+0000 7fe75f539500 2 removed watcher, disabling cache 2023-09-22T13:09:51.850 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.850+0000 7fe75f539500 20 remove_watcher() i=1 2023-09-22T13:09:51.851 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.851+0000 7fe75f539500 20 remove_watcher() i=2 2023-09-22T13:09:51.852 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.851+0000 7fe75f539500 20 remove_watcher() i=3 2023-09-22T13:09:51.853 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.852+0000 7fe75f539500 20 remove_watcher() i=4 2023-09-22T13:09:51.854 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.853+0000 7fe75f539500 20 remove_watcher() i=5 2023-09-22T13:09:51.855 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.854+0000 7fe75f539500 20 remove_watcher() i=6 2023-09-22T13:09:51.855 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.855+0000 7fe75f539500 20 remove_watcher() i=7 2023-09-22T13:09:51.856 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:09:51.855+0000 7fe71bfe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:09:51.860 INFO:tasks.workunit.client.0.smithi098.stderr:{'rgw.main': {'size': 0, 'size_actual': 0, 'size_utilized': 0, 'size_kb': 0, 'size_kb_actual': 0, 'size_kb_utilized': 0, 'num_objects': 0}} 2023-09-22T13:09:51.860 INFO:tasks.workunit.client.0.smithi098.stderr:Deleting bucket check-bucket 2023-09-22T13:09:51.882 INFO:tasks.workunit.client.0.smithi098.stderr:Completed bucket check tests 2023-09-22T13:09:51.883 INFO:tasks.workunit.client.0.smithi098.stdout:connected to http://localhost:80 2023-09-22T13:09:51.918 INFO:tasks.workunit.client.0.smithi098.stderr:+ deactivate 2023-09-22T13:09:51.918 INFO:tasks.workunit.client.0.smithi098.stderr:+ '[' -n /home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin ']' 2023-09-22T13:09:51.918 INFO:tasks.workunit.client.0.smithi098.stderr:+ PATH=/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/sbin 2023-09-22T13:09:51.918 INFO:tasks.workunit.client.0.smithi098.stderr:+ export PATH 2023-09-22T13:09:51.918 INFO:tasks.workunit.client.0.smithi098.stderr:+ unset _OLD_VIRTUAL_PATH 2023-09-22T13:09:51.918 INFO:tasks.workunit.client.0.smithi098.stderr:+ '[' -n '' ']' 2023-09-22T13:09:51.918 INFO:tasks.workunit.client.0.smithi098.stderr:+ '[' -n /usr/bin/bash -o -n '' ']' 2023-09-22T13:09:51.919 INFO:tasks.workunit.client.0.smithi098.stderr:+ hash -r 2023-09-22T13:09:51.919 INFO:tasks.workunit.client.0.smithi098.stderr:+ '[' -n '' ']' 2023-09-22T13:09:51.919 INFO:tasks.workunit.client.0.smithi098.stderr:+ unset VIRTUAL_ENV 2023-09-22T13:09:51.919 INFO:tasks.workunit.client.0.smithi098.stderr:+ '[' '!' '' = nondestructive ']' 2023-09-22T13:09:51.919 INFO:tasks.workunit.client.0.smithi098.stderr:+ unset -f deactivate 2023-09-22T13:09:51.919 INFO:tasks.workunit.client.0.smithi098.stderr:+ echo OK. 2023-09-22T13:09:51.920 INFO:tasks.workunit.client.0.smithi098.stdout:OK. 2023-09-22T13:09:51.920 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:09:51.920 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:09:51.988 INFO:tasks.workunit:Stopping ['rgw/run-bucket-check.sh'] on client.0... 2023-09-22T13:09:51.988 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-22T13:09:52.314 DEBUG:teuthology.parallel:result is None 2023-09-22T13:09:52.315 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:09:52.341 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:09:52.342 DEBUG:teuthology.orchestra.run.smithi098:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:09:52.409 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:09:52.409 INFO:teuthology.run_tasks:Running task workunit... 2023-09-22T13:09:52.418 INFO:tasks.workunit:Pulling workunits from ref 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:09:52.418 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-22T13:09:52.419 DEBUG:teuthology.orchestra.run.smithi098:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:09:52.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-22T13:09:52.467 INFO:teuthology.orchestra.run.smithi098.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-22T13:09:52.468 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:09:52.525 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-22T13:09:52.525 DEBUG:teuthology.orchestra.run.smithi098:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-22T13:09:52.583 INFO:tasks.workunit:timeout=3h 2023-09-22T13:09:52.583 INFO:tasks.workunit:cleanup=True 2023-09-22T13:09:52.584 DEBUG:teuthology.orchestra.run.smithi098:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 31c126b7c1c0c06b66d9d19c5bfc51c46b503392 2023-09-22T13:09:52.642 INFO:tasks.workunit.client.0.smithi098.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-22T13:10:56.962 INFO:tasks.workunit.client.0.smithi098.stderr:Note: switching to '31c126b7c1c0c06b66d9d19c5bfc51c46b503392'. 2023-09-22T13:10:56.963 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:10:56.963 INFO:tasks.workunit.client.0.smithi098.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2023-09-22T13:10:56.963 INFO:tasks.workunit.client.0.smithi098.stderr:changes and commit them, and you can discard any commits you make in this 2023-09-22T13:10:56.963 INFO:tasks.workunit.client.0.smithi098.stderr:state without impacting any branches by switching back to a branch. 2023-09-22T13:10:56.963 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:10:56.964 INFO:tasks.workunit.client.0.smithi098.stderr:If you want to create a new branch to retain commits you create, you may 2023-09-22T13:10:56.964 INFO:tasks.workunit.client.0.smithi098.stderr:do so (now or later) by using -c with the switch command. Example: 2023-09-22T13:10:56.964 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:10:56.964 INFO:tasks.workunit.client.0.smithi098.stderr: git switch -c 2023-09-22T13:10:56.964 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:10:56.965 INFO:tasks.workunit.client.0.smithi098.stderr:Or undo this operation with: 2023-09-22T13:10:56.965 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:10:56.965 INFO:tasks.workunit.client.0.smithi098.stderr: git switch - 2023-09-22T13:10:56.965 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:10:56.965 INFO:tasks.workunit.client.0.smithi098.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2023-09-22T13:10:56.965 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:10:56.965 INFO:tasks.workunit.client.0.smithi098.stderr:HEAD is now at 31c126b7c1c rgw: radosgw-admin bucket check should only print index entries with --check-objects flag 2023-09-22T13:10:56.969 DEBUG:teuthology.orchestra.run.smithi098:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-22T13:10:57.027 INFO:tasks.workunit.client.0.smithi098.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-22T13:10:57.029 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:10:57.029 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-22T13:10:57.075 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-22T13:10:57.121 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-22T13:10:57.159 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-22T13:10:57.160 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:10:57.161 INFO:tasks.workunit.client.0.smithi098.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-22T13:10:57.197 INFO:tasks.workunit.client.0.smithi098.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-22T13:10:57.200 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:10:57.200 DEBUG:teuthology.orchestra.run.smithi098:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-22T13:10:57.258 INFO:tasks.workunit:Running workunits matching cls/test_cls_lock.sh on client.0... 2023-09-22T13:10:57.259 INFO:tasks.workunit:Running workunit cls/test_cls_lock.sh... 2023-09-22T13:10:57.260 DEBUG:teuthology.orchestra.run.smithi098:workunit test cls/test_cls_lock.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_lock.sh 2023-09-22T13:10:57.329 INFO:tasks.workunit.client.0.smithi098.stdout:Running main() from gmock_main.cc 2023-09-22T13:10:57.330 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 11 tests from 1 test suite. 2023-09-22T13:10:57.330 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:10:57.330 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 11 tests from ClsLock 2023-09-22T13:10:57.330 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] ClsLock.TestMultiLocking 2023-09-22T13:10:59.218 INFO:tasks.workunit.client.0.smithi098.stdout:lock: mylock 2023-09-22T13:10:59.218 INFO:tasks.workunit.client.0.smithi098.stdout: lock_type: exclusive 2023-09-22T13:10:59.218 INFO:tasks.workunit.client.0.smithi098.stdout: tag: 2023-09-22T13:10:59.218 INFO:tasks.workunit.client.0.smithi098.stdout: lockers: 2023-09-22T13:10:59.219 INFO:tasks.workunit.client.0.smithi098.stdout: client.4784 expiration=2023-09-22T13:12:59.215768+0000 addr=v1:172.21.15.98:0/3146682988 cookie= 2023-09-22T13:10:59.221 INFO:tasks.workunit.client.0.smithi098.stdout:lock: mylock 2023-09-22T13:10:59.221 INFO:tasks.workunit.client.0.smithi098.stdout: lock_type: shared 2023-09-22T13:10:59.221 INFO:tasks.workunit.client.0.smithi098.stdout: tag: 2023-09-22T13:10:59.221 INFO:tasks.workunit.client.0.smithi098.stdout: lockers: 2023-09-22T13:10:59.221 INFO:tasks.workunit.client.0.smithi098.stdout: client.4784 expiration=2023-09-22T13:12:59.220983+0000 addr=v1:172.21.15.98:0/3146682988 cookie= 2023-09-22T13:10:59.221 INFO:tasks.workunit.client.0.smithi098.stdout: client.4938 expiration=0.000000 addr=v1:172.21.15.98:0/1330356418 cookie= 2023-09-22T13:10:59.222 INFO:tasks.workunit.client.0.smithi098.stdout:lock: mylock 2023-09-22T13:10:59.222 INFO:tasks.workunit.client.0.smithi098.stdout: lock_type: shared 2023-09-22T13:10:59.222 INFO:tasks.workunit.client.0.smithi098.stdout: tag: 2023-09-22T13:10:59.222 INFO:tasks.workunit.client.0.smithi098.stdout: lockers: 2023-09-22T13:10:59.222 INFO:tasks.workunit.client.0.smithi098.stdout: client.4938 expiration=0.000000 addr=v1:172.21.15.98:0/1330356418 cookie= 2023-09-22T13:11:00.157 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] ClsLock.TestMultiLocking (2828 ms) 2023-09-22T13:11:00.157 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] ClsLock.TestMeta 2023-09-22T13:11:02.200 INFO:tasks.workunit.client.0.smithi098.stdout:lock: mylock 2023-09-22T13:11:02.200 INFO:tasks.workunit.client.0.smithi098.stdout: lock_type: shared 2023-09-22T13:11:02.200 INFO:tasks.workunit.client.0.smithi098.stdout: tag: 2023-09-22T13:11:02.200 INFO:tasks.workunit.client.0.smithi098.stdout: lockers: 2023-09-22T13:11:02.200 INFO:tasks.workunit.client.0.smithi098.stdout: client.4953 expiration=0.000000 addr=v1:172.21.15.98:0/3181065811 cookie= 2023-09-22T13:11:02.202 INFO:tasks.workunit.client.0.smithi098.stdout:lock: mylock 2023-09-22T13:11:02.202 INFO:tasks.workunit.client.0.smithi098.stdout: lock_type: exclusive 2023-09-22T13:11:02.202 INFO:tasks.workunit.client.0.smithi098.stdout: tag: new_tag 2023-09-22T13:11:02.202 INFO:tasks.workunit.client.0.smithi098.stdout: lockers: 2023-09-22T13:11:02.202 INFO:tasks.workunit.client.0.smithi098.stdout: client.4793 expiration=0.000000 addr=v1:172.21.15.98:0/3013095231 cookie= 2023-09-22T13:11:03.168 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] ClsLock.TestMeta (3011 ms) 2023-09-22T13:11:03.168 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] ClsLock.TestCookie 2023-09-22T13:11:05.216 INFO:tasks.workunit.client.0.smithi098.stdout:lock: mylock 2023-09-22T13:11:05.216 INFO:tasks.workunit.client.0.smithi098.stdout: lock_type: exclusive 2023-09-22T13:11:05.217 INFO:tasks.workunit.client.0.smithi098.stdout: tag: 2023-09-22T13:11:05.217 INFO:tasks.workunit.client.0.smithi098.stdout: lockers: 2023-09-22T13:11:05.218 INFO:tasks.workunit.client.0.smithi098.stdout:lock: mylock 2023-09-22T13:11:05.218 INFO:tasks.workunit.client.0.smithi098.stdout: lock_type: shared 2023-09-22T13:11:05.218 INFO:tasks.workunit.client.0.smithi098.stdout: tag: 2023-09-22T13:11:05.219 INFO:tasks.workunit.client.0.smithi098.stdout: lockers: 2023-09-22T13:11:05.219 INFO:tasks.workunit.client.0.smithi098.stdout: client.4959 expiration=0.000000 addr=v1:172.21.15.98:0/747923555 cookie= 2023-09-22T13:11:05.219 INFO:tasks.workunit.client.0.smithi098.stdout: client.4959 expiration=0.000000 addr=v1:172.21.15.98:0/747923555 cookie=new cookie 2023-09-22T13:11:06.200 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] ClsLock.TestCookie (3031 ms) 2023-09-22T13:11:06.200 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] ClsLock.TestMultipleLocks 2023-09-22T13:11:09.223 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] ClsLock.TestMultipleLocks (3024 ms) 2023-09-22T13:11:09.223 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] ClsLock.TestLockDuration 2023-09-22T13:11:17.261 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] ClsLock.TestLockDuration (8037 ms) 2023-09-22T13:11:17.261 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] ClsLock.TestAssertLocked 2023-09-22T13:11:20.284 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] ClsLock.TestAssertLocked (3023 ms) 2023-09-22T13:11:20.285 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] ClsLock.TestSetCookie 2023-09-22T13:11:23.305 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] ClsLock.TestSetCookie (3022 ms) 2023-09-22T13:11:23.305 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] ClsLock.TestRenew 2023-09-22T13:11:44.077 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] ClsLock.TestRenew (20771 ms) 2023-09-22T13:11:44.077 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralBasic 2023-09-22T13:11:51.128 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] ClsLock.TestExclusiveEphemeralBasic (7052 ms) 2023-09-22T13:11:51.128 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealEphemeral 2023-09-22T13:11:59.158 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealEphemeral (8029 ms) 2023-09-22T13:11:59.158 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealExclusive 2023-09-22T13:12:07.205 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealExclusive (8047 ms) 2023-09-22T13:12:07.205 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 11 tests from ClsLock (69875 ms total) 2023-09-22T13:12:07.205 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:12:07.205 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:12:07.205 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 11 tests from 1 test suite ran. (69875 ms total) 2023-09-22T13:12:07.205 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 11 tests. 2023-09-22T13:12:07.207 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:12:07.207 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:12:07.282 INFO:tasks.workunit:Running workunits matching cls/test_cls_log.sh on client.0... 2023-09-22T13:12:07.283 INFO:tasks.workunit:Running workunit cls/test_cls_log.sh... 2023-09-22T13:12:07.283 DEBUG:teuthology.orchestra.run.smithi098:workunit test cls/test_cls_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_log.sh 2023-09-22T13:12:07.357 INFO:tasks.workunit.client.0.smithi098.stdout:Running main() from gmock_main.cc 2023-09-22T13:12:07.357 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 4 tests from 1 test suite. 2023-09-22T13:12:07.357 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:12:07.357 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 4 tests from cls_log 2023-09-22T13:12:07.357 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_log.test_log_add_same_time 2023-09-22T13:12:10.219 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_log.test_log_add_same_time (2862 ms) 2023-09-22T13:12:10.219 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_log.test_log_add_different_time 2023-09-22T13:12:13.243 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_log.test_log_add_different_time (3025 ms) 2023-09-22T13:12:13.243 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_log.trim_by_time 2023-09-22T13:12:17.277 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_log.trim_by_time (4034 ms) 2023-09-22T13:12:17.277 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_log.trim_by_marker 2023-09-22T13:12:20.307 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_log.trim_by_marker (3029 ms) 2023-09-22T13:12:20.307 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 4 tests from cls_log (12950 ms total) 2023-09-22T13:12:20.307 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:12:20.307 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:12:20.307 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 4 tests from 1 test suite ran. (12950 ms total) 2023-09-22T13:12:20.307 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 4 tests. 2023-09-22T13:12:20.309 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:12:20.309 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:12:20.380 INFO:tasks.workunit:Running workunits matching cls/test_cls_refcount.sh on client.0... 2023-09-22T13:12:20.381 INFO:tasks.workunit:Running workunit cls/test_cls_refcount.sh... 2023-09-22T13:12:20.381 DEBUG:teuthology.orchestra.run.smithi098:workunit test cls/test_cls_refcount.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_refcount.sh 2023-09-22T13:12:20.451 INFO:tasks.workunit.client.0.smithi098.stdout:Running main() from gmock_main.cc 2023-09-22T13:12:20.452 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 10 tests from 1 test suite. 2023-09-22T13:12:20.452 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:12:20.452 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 10 tests from cls_rgw 2023-09-22T13:12:20.452 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.test_implicit 2023-09-22T13:12:23.314 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.test_implicit (2863 ms) 2023-09-22T13:12:23.315 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.test_implicit_idempotent 2023-09-22T13:12:26.350 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.test_implicit_idempotent (3036 ms) 2023-09-22T13:12:26.351 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.test_put_snap 2023-09-22T13:12:35.424 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.test_put_snap (9074 ms) 2023-09-22T13:12:35.424 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.test_explicit 2023-09-22T13:12:38.430 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.test_explicit (3006 ms) 2023-09-22T13:12:38.430 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.set 2023-09-22T13:12:41.448 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.set (3017 ms) 2023-09-22T13:12:41.448 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.test_implicit_ec 2023-09-22T13:12:48.498 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.test_implicit_ec (7051 ms) 2023-09-22T13:12:48.499 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.test_implicit_idempotent_ec 2023-09-22T13:12:55.522 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.test_implicit_idempotent_ec (7023 ms) 2023-09-22T13:12:55.522 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.test_put_snap_ec 2023-09-22T13:13:07.588 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.test_put_snap_ec (12067 ms) 2023-09-22T13:13:07.588 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.test_explicit_ec 2023-09-22T13:13:14.645 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.test_explicit_ec (7057 ms) 2023-09-22T13:13:14.645 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.set_ec 2023-09-22T13:13:21.685 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.set_ec (7039 ms) 2023-09-22T13:13:21.685 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 10 tests from cls_rgw (61233 ms total) 2023-09-22T13:13:21.685 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:13:21.685 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:13:21.685 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 10 tests from 1 test suite ran. (61233 ms total) 2023-09-22T13:13:21.685 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 10 tests. 2023-09-22T13:13:21.687 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:13:21.687 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:13:21.759 INFO:tasks.workunit:Running workunits matching cls/test_cls_rgw.sh on client.0... 2023-09-22T13:13:21.760 INFO:tasks.workunit:Running workunit cls/test_cls_rgw.sh... 2023-09-22T13:13:21.760 DEBUG:teuthology.orchestra.run.smithi098:workunit test cls/test_cls_rgw.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rgw.sh 2023-09-22T13:13:21.833 INFO:tasks.workunit.client.0.smithi098.stdout:Running main() from gmock_main.cc 2023-09-22T13:13:21.833 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 16 tests from 1 test suite. 2023-09-22T13:13:21.834 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:13:21.834 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 16 tests from cls_rgw 2023-09-22T13:13:23.700 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.index_basic 2023-09-22T13:13:23.761 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.index_basic (60 ms) 2023-09-22T13:13:23.761 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.index_multiple_obj_writers 2023-09-22T13:13:23.783 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.index_multiple_obj_writers (23 ms) 2023-09-22T13:13:23.783 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.index_remove_object 2023-09-22T13:13:23.814 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.index_remove_object (30 ms) 2023-09-22T13:13:23.814 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.index_suggest 2023-09-22T13:13:25.196 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.index_suggest (1383 ms) 2023-09-22T13:13:25.197 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.index_suggest_complete 2023-09-22T13:13:25.206 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.index_suggest_complete (10 ms) 2023-09-22T13:13:25.206 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.index_list 2023-09-22T13:13:25.224 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.index_list (17 ms) 2023-09-22T13:13:25.224 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.index_list_delimited 2023-09-22T13:13:44.470 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.index_list_delimited (19247 ms) 2023-09-22T13:13:44.470 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.bi_list 2023-09-22T13:13:44.549 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.bi_list (78 ms) 2023-09-22T13:13:44.549 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.gc_set 2023-09-22T13:13:44.559 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.gc_set (11 ms) 2023-09-22T13:13:44.559 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.gc_list 2023-09-22T13:13:44.570 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.gc_list (11 ms) 2023-09-22T13:13:44.571 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.gc_defer 2023-09-22T13:13:52.161 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.gc_defer (7590 ms) 2023-09-22T13:13:52.161 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.usage_basic 2023-09-22T13:13:52.301 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.usage_basic (141 ms) 2023-09-22T13:13:52.302 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.usage_clear_no_obj 2023-09-22T13:13:52.302 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.usage_clear_no_obj (1 ms) 2023-09-22T13:13:52.302 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.usage_clear 2023-09-22T13:13:52.381 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.usage_clear (78 ms) 2023-09-22T13:13:52.381 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.bi_log_trim 2023-09-22T13:13:52.421 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.bi_log_trim (41 ms) 2023-09-22T13:13:52.421 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] cls_rgw.index_racing_removes 2023-09-22T13:13:52.430 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] cls_rgw.index_racing_removes (9 ms) 2023-09-22T13:13:53.173 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 16 tests from cls_rgw (28730 ms total) 2023-09-22T13:13:53.173 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:13:53.173 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:13:53.173 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 16 tests from 1 test suite ran. (31339 ms total) 2023-09-22T13:13:53.173 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 16 tests. 2023-09-22T13:13:53.175 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:13:53.175 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:13:53.246 INFO:tasks.workunit:Running workunits matching cls/test_cls_cmpomap.sh on client.0... 2023-09-22T13:13:53.247 INFO:tasks.workunit:Running workunit cls/test_cls_cmpomap.sh... 2023-09-22T13:13:53.247 DEBUG:teuthology.orchestra.run.smithi098:workunit test cls/test_cls_cmpomap.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cmpomap.sh 2023-09-22T13:13:53.319 INFO:tasks.workunit.client.0.smithi098.stdout:Running main() from gmock_main.cc 2023-09-22T13:13:53.319 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 31 tests from 1 test suite. 2023-09-22T13:13:53.319 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:13:55.178 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 31 tests from CmpOmap 2023-09-22T13:13:55.178 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str 2023-09-22T13:13:55.206 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str (29 ms) 2023-09-22T13:13:55.206 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str_default 2023-09-22T13:13:55.211 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str_default (5 ms) 2023-09-22T13:13:55.211 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64 2023-09-22T13:13:55.213 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64 (2 ms) 2023-09-22T13:13:55.214 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64_default 2023-09-22T13:13:55.217 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64_default (4 ms) 2023-09-22T13:13:55.217 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_str 2023-09-22T13:13:55.223 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_str (6 ms) 2023-09-22T13:13:55.223 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_u64 2023-09-22T13:13:55.230 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_u64 (6 ms) 2023-09-22T13:13:55.230 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_input 2023-09-22T13:13:55.233 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_input (3 ms) 2023-09-22T13:13:55.233 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_default 2023-09-22T13:13:55.237 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_default (4 ms) 2023-09-22T13:13:55.237 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_u64_empty_default 2023-09-22T13:13:55.239 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_u64_empty_default (2 ms) 2023-09-22T13:13:55.240 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_value 2023-09-22T13:13:55.243 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_value (4 ms) 2023-09-22T13:13:55.243 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_at_max_keys 2023-09-22T13:13:55.244 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_at_max_keys (1 ms) 2023-09-22T13:13:55.244 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_vals_over_max_keys 2023-09-22T13:13:55.244 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_vals_over_max_keys (0 ms) 2023-09-22T13:13:55.244 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str 2023-09-22T13:13:55.249 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str (5 ms) 2023-09-22T13:13:55.249 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str_default 2023-09-22T13:13:55.254 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str_default (5 ms) 2023-09-22T13:13:55.254 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64 2023-09-22T13:13:55.258 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64 (4 ms) 2023-09-22T13:13:55.258 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64_default 2023-09-22T13:13:55.264 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64_default (6 ms) 2023-09-22T13:13:55.264 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_set_vals_str 2023-09-22T13:13:55.270 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_set_vals_str (5 ms) 2023-09-22T13:13:55.270 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64 2023-09-22T13:13:55.277 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_set_vals_u64 (8 ms) 2023-09-22T13:13:55.277 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_einval 2023-09-22T13:13:55.279 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_einval (1 ms) 2023-09-22T13:13:55.279 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_eio 2023-09-22T13:13:55.281 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_eio (2 ms) 2023-09-22T13:13:55.281 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_set_vals_at_max_keys 2023-09-22T13:13:55.282 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_set_vals_at_max_keys (0 ms) 2023-09-22T13:13:55.282 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_set_vals_over_max_keys 2023-09-22T13:13:55.282 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_set_vals_over_max_keys (1 ms) 2023-09-22T13:13:55.282 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_str 2023-09-22T13:13:55.286 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_str (4 ms) 2023-09-22T13:13:55.287 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_u64 2023-09-22T13:13:55.291 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_u64 (5 ms) 2023-09-22T13:13:55.291 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_rm_keys_str 2023-09-22T13:13:55.296 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_rm_keys_str (5 ms) 2023-09-22T13:13:55.296 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64 2023-09-22T13:13:55.302 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64 (6 ms) 2023-09-22T13:13:55.302 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_einval 2023-09-22T13:13:55.304 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_einval (2 ms) 2023-09-22T13:13:55.304 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_eio 2023-09-22T13:13:55.306 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_eio (2 ms) 2023-09-22T13:13:55.306 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_rm_keys_at_max_keys 2023-09-22T13:13:55.307 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_rm_keys_at_max_keys (1 ms) 2023-09-22T13:13:55.307 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_rm_keys_over_max_keys 2023-09-22T13:13:55.308 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_rm_keys_over_max_keys (0 ms) 2023-09-22T13:13:55.308 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_empty 2023-09-22T13:13:55.314 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_empty (7 ms) 2023-09-22T13:13:55.314 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 31 tests from CmpOmap (137 ms total) 2023-09-22T13:13:55.315 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:13:55.315 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:13:56.178 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 31 tests from 1 test suite ran. (2859 ms total) 2023-09-22T13:13:56.179 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 31 tests. 2023-09-22T13:13:56.181 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:13:56.182 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:13:56.251 INFO:tasks.workunit:Running workunits matching cls/test_cls_2pc_queue.sh on client.0... 2023-09-22T13:13:56.252 INFO:tasks.workunit:Running workunit cls/test_cls_2pc_queue.sh... 2023-09-22T13:13:56.253 DEBUG:teuthology.orchestra.run.smithi098:workunit test cls/test_cls_2pc_queue.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh 2023-09-22T13:13:56.324 INFO:tasks.workunit.client.0.smithi098.stdout:Running main() from gmock_main.cc 2023-09-22T13:13:56.324 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 22 tests from 1 test suite. 2023-09-22T13:13:56.324 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:13:56.324 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 22 tests from TestCls2PCQueue 2023-09-22T13:13:56.324 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.GetCapacity 2023-09-22T13:13:59.217 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.GetCapacity (2892 ms) 2023-09-22T13:13:59.217 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.AsyncGetCapacity 2023-09-22T13:14:02.236 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.AsyncGetCapacity (3020 ms) 2023-09-22T13:14:02.236 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.Reserve 2023-09-22T13:14:05.265 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.Reserve (3029 ms) 2023-09-22T13:14:05.266 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.AsyncReserve 2023-09-22T13:14:08.285 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.AsyncReserve (3020 ms) 2023-09-22T13:14:08.285 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.Commit 2023-09-22T13:14:11.300 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.Commit (3015 ms) 2023-09-22T13:14:11.300 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.Abort 2023-09-22T13:14:14.346 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.Abort (3046 ms) 2023-09-22T13:14:14.347 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.ReserveError 2023-09-22T13:14:17.352 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.ReserveError (3006 ms) 2023-09-22T13:14:17.352 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.CommitError 2023-09-22T13:14:20.384 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.CommitError (3032 ms) 2023-09-22T13:14:20.384 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.AbortError 2023-09-22T13:14:23.399 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.AbortError (3015 ms) 2023-09-22T13:14:23.400 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.MultiReserve 2023-09-22T13:14:26.431 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.MultiReserve (3032 ms) 2023-09-22T13:14:26.431 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.MultiCommit 2023-09-22T13:14:29.459 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.MultiCommit (3027 ms) 2023-09-22T13:14:29.460 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.MultiAbort 2023-09-22T13:14:32.483 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.MultiAbort (3024 ms) 2023-09-22T13:14:32.483 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.ReserveCommit 2023-09-22T13:14:35.499 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.ReserveCommit (3017 ms) 2023-09-22T13:14:35.500 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.ReserveAbort 2023-09-22T13:14:38.511 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.ReserveAbort (3009 ms) 2023-09-22T13:14:38.512 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.ManualCleanup 2023-09-22T13:14:41.528 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.ManualCleanup (3019 ms) 2023-09-22T13:14:41.528 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.Cleanup 2023-09-22T13:14:44.555 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.Cleanup (3028 ms) 2023-09-22T13:14:44.555 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.MultiProducer 2023-09-22T13:14:55.099 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.MultiProducer (10543 ms) 2023-09-22T13:14:55.099 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.AsyncConsumer 2023-09-22T13:14:58.114 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.AsyncConsumer (3015 ms) 2023-09-22T13:14:58.114 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.MultiProducerConsumer 2023-09-22T13:15:11.152 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.MultiProducerConsumer (13038 ms) 2023-09-22T13:15:11.152 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.ReserveSpillover 2023-09-22T13:15:14.173 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.ReserveSpillover (3021 ms) 2023-09-22T13:15:14.173 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.CommitSpillover 2023-09-22T13:15:18.211 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.CommitSpillover (4039 ms) 2023-09-22T13:15:18.211 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestCls2PCQueue.AbortSpillover 2023-09-22T13:15:22.260 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestCls2PCQueue.AbortSpillover (4046 ms) 2023-09-22T13:15:22.260 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 22 tests from TestCls2PCQueue (85933 ms total) 2023-09-22T13:15:22.260 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:22.261 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:22.261 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 22 tests from 1 test suite ran. (85933 ms total) 2023-09-22T13:15:22.261 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 22 tests. 2023-09-22T13:15:22.261 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:15:22.261 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:15:22.333 INFO:tasks.workunit:Running workunits matching rgw/test_rgw_gc_log.sh on client.0... 2023-09-22T13:15:22.335 INFO:tasks.workunit:Running workunit rgw/test_rgw_gc_log.sh... 2023-09-22T13:15:22.335 DEBUG:teuthology.orchestra.run.smithi098:workunit test rgw/test_rgw_gc_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_gc_log.sh 2023-09-22T13:15:22.414 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:22.414 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:22.414 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 6 tests from 1 test suite. 2023-09-22T13:15:22.414 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:24.286 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 6 tests from rgw_gc_log 2023-09-22T13:15:24.286 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] rgw_gc_log.init_existing_queue 2023-09-22T13:15:24.303 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] rgw_gc_log.init_existing_queue (18 ms) 2023-09-22T13:15:24.304 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] rgw_gc_log.init_existing_omap 2023-09-22T13:15:24.307 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] rgw_gc_log.init_existing_omap (4 ms) 2023-09-22T13:15:24.307 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] rgw_gc_log.enqueue1_after_init 2023-09-22T13:15:24.310 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] rgw_gc_log.enqueue1_after_init (2 ms) 2023-09-22T13:15:24.310 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] rgw_gc_log.enqueue2_before_init 2023-09-22T13:15:24.311 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] rgw_gc_log.enqueue2_before_init (2 ms) 2023-09-22T13:15:24.311 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] rgw_gc_log.defer1_after_init 2023-09-22T13:15:24.313 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] rgw_gc_log.defer1_after_init (2 ms) 2023-09-22T13:15:24.314 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] rgw_gc_log.defer2_before_init 2023-09-22T13:15:24.314 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] rgw_gc_log.defer2_before_init (1 ms) 2023-09-22T13:15:24.314 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 6 tests from rgw_gc_log (29 ms total) 2023-09-22T13:15:24.314 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:24.315 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:25.294 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 6 tests from 1 test suite ran. (2880 ms total) 2023-09-22T13:15:25.294 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 6 tests. 2023-09-22T13:15:25.297 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:15:25.297 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:15:25.370 INFO:tasks.workunit:Running workunits matching rgw/test_rgw_obj.sh on client.0... 2023-09-22T13:15:25.371 INFO:tasks.workunit:Running workunit rgw/test_rgw_obj.sh... 2023-09-22T13:15:25.371 DEBUG:teuthology.orchestra.run.smithi098:workunit test rgw/test_rgw_obj.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_obj.sh 2023-09-22T13:15:25.450 INFO:tasks.workunit.client.0.smithi098.stdout:Running main() from gmock_main.cc 2023-09-22T13:15:25.451 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 4 tests from 1 test suite. 2023-09-22T13:15:25.451 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:25.451 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 4 tests from TestRGWObj 2023-09-22T13:15:25.451 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestRGWObj.underscore 2023-09-22T13:15:25.451 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.451 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.451 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns= i= 2023-09-22T13:15:25.451 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.451 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.451 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns= i= 2023-09-22T13:15:25.451 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns= i= 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout: "obj1": { 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.452 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.453 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.453 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.453 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.453 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.453 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.453 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.453 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.453 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.453 INFO:tasks.workunit.client.0.smithi098.stdout: "obj2": { 2023-09-22T13:15:25.453 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.453 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.454 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3": { 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.455 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3-2": { 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.456 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.457 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns=ns i= 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns=ns i= 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj 2023-09-22T13:15:25.458 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns=ns i= 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: "obj1": { 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.459 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "obj2": { 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.460 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3": { 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.461 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.462 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3-2": { 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.463 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns= i=v1 2023-09-22T13:15:25.464 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns= i=v1 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns= i=v1 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout: "obj1": { 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.465 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.466 INFO:tasks.workunit.client.0.smithi098.stdout: "obj2": { 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.467 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3": { 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.468 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3-2": { 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.469 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.470 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns=ns i=v1 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns=ns i=v1 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: _obj[v1] 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: _obj ns=ns i=v1 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout: "obj1": { 2023-09-22T13:15:25.471 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.472 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: "obj2": { 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.473 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3": { 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.474 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.475 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3-2": { 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.476 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_obj", 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestRGWObj.underscore (0 ms) 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestRGWObj.no_underscore 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.477 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns= i= 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns= i= 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns= i= 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout: "obj1": { 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.478 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.479 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: "obj2": { 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.480 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3": { 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.481 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.482 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3-2": { 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.483 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.484 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.484 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.484 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.484 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.484 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.484 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.484 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.484 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.484 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns=ns i= 2023-09-22T13:15:25.484 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.484 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.485 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns=ns i= 2023-09-22T13:15:25.485 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.485 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj 2023-09-22T13:15:25.485 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns=ns i= 2023-09-22T13:15:25.485 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.485 INFO:tasks.workunit.client.0.smithi098.stdout: "obj1": { 2023-09-22T13:15:25.485 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.485 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.485 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.486 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.486 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.486 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.486 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.486 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.486 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.486 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.486 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.486 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.486 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.486 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: "obj2": { 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.487 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3": { 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.488 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.489 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.490 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3-2": { 2023-09-22T13:15:25.490 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.490 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.490 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.490 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.490 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.490 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.490 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.490 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.490 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.490 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns= i=v1 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.491 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns= i=v1 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns= i=v1 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout: "obj1": { 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.492 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: "obj2": { 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.493 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.494 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3": { 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.495 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3-2": { 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.496 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.497 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns=ns i=v1 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns=ns i=v1 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout:parsed: obj[v1] 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout:stripped: obj ns=ns i=v1 2023-09-22T13:15:25.498 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: "obj1": { 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.499 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "obj2": { 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.500 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3": { 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.501 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.502 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: "obj3-2": { 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.503 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "obj", 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "v1", 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestRGWObj.no_underscore (0 ms) 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestRGWObj.obj_to_raw 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.504 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.505 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.506 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker_myobj", 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.507 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.508 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.509 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.510 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.511 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker_myobj", 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.512 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.513 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.514 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.515 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.516 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.517 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.518 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.518 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.518 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.518 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.518 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.518 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.518 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.518 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.518 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.518 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.518 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.519 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.520 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.521 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.522 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.523 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.524 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.525 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.526 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.527 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.528 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.529 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.530 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.531 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.532 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.533 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.534 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.535 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.536 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.537 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.538 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.539 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.540 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.541 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.542 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.542 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.542 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.542 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.542 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.542 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.542 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.542 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.542 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.542 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.542 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.543 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker___myobj", 2023-09-22T13:15:25.544 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj" 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.545 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.546 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.546 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.546 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.546 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.546 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.546 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.546 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.546 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.546 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.546 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.546 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.547 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.547 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.547 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.547 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.547 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.547 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.547 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.547 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.547 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.547 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.547 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.548 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.549 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker___myobj", 2023-09-22T13:15:25.550 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj" 2023-09-22T13:15:25.550 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.550 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.550 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.550 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.550 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.550 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.550 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.550 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.550 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.550 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.551 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.552 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.553 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:15:25.554 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.555 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.556 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.557 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.558 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.559 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.560 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.561 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.562 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.563 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.564 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.564 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.564 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.564 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.564 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.564 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.564 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:15:25.564 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj" 2023-09-22T13:15:25.564 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.564 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.564 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.565 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.566 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.567 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.567 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.567 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.567 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.567 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.567 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.567 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.567 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.567 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.567 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.567 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.568 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj" 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.569 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.570 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.571 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.571 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.571 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.571 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.571 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.571 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.571 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.571 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.571 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.571 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.571 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.572 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.573 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.574 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.574 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.574 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.574 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.574 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.574 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.574 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.574 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.574 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:15:25.574 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.574 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.575 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.576 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.576 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.576 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.576 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.576 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.576 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.576 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.576 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.576 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.576 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.576 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.577 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.578 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.578 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.578 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.578 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.578 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.578 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.578 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.578 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.578 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.578 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.578 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.579 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.580 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.581 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.582 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.583 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker___myobj_", 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj_" 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.584 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.585 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.586 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.587 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.588 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker___myobj_", 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj_" 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.589 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.590 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.591 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.592 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.593 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.594 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.595 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.596 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.597 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.598 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.599 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.600 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.601 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.602 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj_" 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.603 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.604 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.604 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.604 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.604 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.604 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.604 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.604 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.604 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.604 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.604 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.604 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.605 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.606 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.606 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.606 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.606 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.606 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.606 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.606 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.606 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.606 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.606 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.606 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.607 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.608 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.608 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.608 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.608 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.608 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.608 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.608 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.608 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.608 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.608 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:15:25.608 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj_" 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.609 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.610 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.610 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.610 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.610 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.610 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.610 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.610 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.610 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.610 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.610 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.610 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.611 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.612 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": "default-placement.data", 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.613 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "test", 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": "", 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.614 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": "" 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.615 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.616 INFO:tasks.workunit.client.0.smithi098.stdout: "obj": { 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.617 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.618 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.619 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestRGWObj.obj_to_raw (1 ms) 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] TestRGWObj.old_to_raw 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.620 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker_myobj", 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.621 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.622 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker_myobj 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker_myobj", 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.623 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.624 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker_myobj 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.625 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.626 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker__ns_myobj 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns_myobj", 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.627 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.628 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.629 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.629 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.629 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.629 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.629 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.629 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.629 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.629 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.629 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.629 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker__ns_myobj 2023-09-22T13:15:25.629 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.630 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.631 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker__:inst_myobj 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst_myobj", 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.632 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.633 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker__:inst_myobj 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.634 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.635 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker__ns:inst_myobj 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst_myobj", 2023-09-22T13:15:25.636 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.637 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "myobj", 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker__ns:inst_myobj 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.638 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker___myobj", 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj" 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.639 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.640 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker___myobj 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker___myobj", 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj" 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.641 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.642 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker___myobj 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.643 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.644 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker__ns__myobj 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns__myobj", 2023-09-22T13:15:25.645 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.646 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker__ns__myobj 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.647 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj" 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.648 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.649 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.650 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.650 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker__:inst__myobj 2023-09-22T13:15:25.650 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.650 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.650 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.650 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst__myobj", 2023-09-22T13:15:25.650 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj" 2023-09-22T13:15:25.650 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.650 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.650 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.650 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.651 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker__:inst__myobj 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.652 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.653 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.654 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.654 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.654 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.654 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.654 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.654 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.654 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.654 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.654 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker__ns:inst__myobj 2023-09-22T13:15:25.654 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.654 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst__myobj", 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.655 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj", 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.656 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker__ns:inst__myobj 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker___myobj_", 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj_" 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.657 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.658 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.658 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.658 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.658 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.658 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.658 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.658 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.658 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.658 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.658 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.658 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker___myobj_ 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker___myobj_", 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj_" 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.659 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.660 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker___myobj_ 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.661 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.662 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker__ns__myobj_ 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.663 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns__myobj_", 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.664 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "", 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.665 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker__ns__myobj_ 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj_" 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.666 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.667 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker__:inst__myobj_ 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__:inst__myobj_", 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "marker__myobj_" 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.668 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.669 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.670 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.670 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "" 2023-09-22T13:15:25.670 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.670 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.670 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.670 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker__:inst__myobj_ 2023-09-22T13:15:25.670 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.670 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj": { 2023-09-22T13:15:25.670 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.670 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:15:25.670 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj": { 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.671 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout:raw=.data-pool:marker__ns:inst__myobj_ 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.672 INFO:tasks.workunit.client.0.smithi098.stdout: "raw_obj2": { 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout: "pool": ".data-pool", 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout: "oid": "marker__ns:inst__myobj_", 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout: "loc": "" 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout: "new_obj2": { 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket": { 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "ebtest", 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout: "marker": "marker", 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_id": "bucket-id", 2023-09-22T13:15:25.673 INFO:tasks.workunit.client.0.smithi098.stdout: "tenant": "", 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: "explicit_placement": { 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: "data_pool": ".data-pool", 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: "data_extra_pool": "", 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: "index_pool": ".index-pool" 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: "key": { 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: "name": "_myobj_", 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: "instance": "inst", 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: "ns": "ns" 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.674 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:25.675 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:25.675 INFO:tasks.workunit.client.0.smithi098.stdout:raw2=.data-pool:marker__ns:inst__myobj_ 2023-09-22T13:15:25.675 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] TestRGWObj.old_to_raw (0 ms) 2023-09-22T13:15:25.675 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 4 tests from TestRGWObj (1 ms total) 2023-09-22T13:15:25.675 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:25.675 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:25.675 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 4 tests from 1 test suite ran. (1 ms total) 2023-09-22T13:15:25.675 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 4 tests. 2023-09-22T13:15:25.675 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:15:25.675 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:15:25.706 INFO:tasks.workunit:Running workunits matching rgw/test_rgw_throttle.sh on client.0... 2023-09-22T13:15:25.707 INFO:tasks.workunit:Running workunit rgw/test_rgw_throttle.sh... 2023-09-22T13:15:25.707 DEBUG:teuthology.orchestra.run.smithi098:workunit test rgw/test_rgw_throttle.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_rgw_throttle.sh 2023-09-22T13:15:25.789 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:25.789 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:25.789 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 5 tests from 1 test suite. 2023-09-22T13:15:25.789 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:27.307 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 5 tests from Aio_Throttle 2023-09-22T13:15:27.308 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] Aio_Throttle.NoThrottleUpToMax 2023-09-22T13:15:27.308 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] Aio_Throttle.NoThrottleUpToMax (0 ms) 2023-09-22T13:15:27.308 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] Aio_Throttle.CostOverWindow 2023-09-22T13:15:27.308 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] Aio_Throttle.CostOverWindow (0 ms) 2023-09-22T13:15:27.308 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] Aio_Throttle.ThrottleOverMax 2023-09-22T13:15:27.395 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] Aio_Throttle.ThrottleOverMax (88 ms) 2023-09-22T13:15:27.396 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] Aio_Throttle.YieldCostOverWindow 2023-09-22T13:15:27.396 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] Aio_Throttle.YieldCostOverWindow (0 ms) 2023-09-22T13:15:27.396 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] Aio_Throttle.YieldingThrottleOverMax 2023-09-22T13:15:27.482 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] Aio_Throttle.YieldingThrottleOverMax (87 ms) 2023-09-22T13:15:27.482 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 5 tests from Aio_Throttle (175 ms total) 2023-09-22T13:15:27.483 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:27.483 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:27.484 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 5 tests from 1 test suite ran. (1694 ms total) 2023-09-22T13:15:27.484 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 5 tests. 2023-09-22T13:15:27.487 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:15:27.487 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:15:27.558 INFO:tasks.workunit:Running workunits matching rgw/test_librgw_file.sh on client.0... 2023-09-22T13:15:27.580 INFO:tasks.workunit:Running workunit rgw/test_librgw_file.sh... 2023-09-22T13:15:27.580 DEBUG:teuthology.orchestra.run.smithi098:workunit test rgw/test_librgw_file.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=31c126b7c1c0c06b66d9d19c5bfc51c46b503392 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rgw/test_librgw_file.sh 2023-09-22T13:15:27.669 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:27.669 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:27.677 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.676+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.679 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.678+0000 7f00a8cba500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:27.680 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.679+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.680 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.679+0000 7f00a8cba500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:27.680 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.679+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.680 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.680+0000 7f00a8cba500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:27.680 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.680+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.681 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.680+0000 7f00a8cba500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:27.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.691+0000 7f00a8cba500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:15:27.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.691+0000 7f00a8cba500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:15:27.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.691+0000 7f00a8cba500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:15:27.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.691+0000 7f00a8cba500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:15:27.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.691+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.691+0000 7f00a8cba500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:27.692 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.691+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.691+0000 7f00a8cba500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:15:27.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.691+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.692+0000 7f00a8cba500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:27.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.692+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.693 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.692+0000 7f00a8cba500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:15:27.694 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.692+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.694 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.693+0000 7f00a8cba500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:27.694 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.693+0000 7f00a8cba500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:15:27.694 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.693+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.694 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.693+0000 7f00a8cba500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:27.695 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.693+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.695 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.693+0000 7f00a8cba500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:27.695 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.693+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.694+0000 7f00a8cba500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:15:27.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.694+0000 7f00a8cba500 20 zonegroup default 2023-09-22T13:15:27.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.694+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.694+0000 7f00a8cba500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:27.696 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.694+0000 7f00a8cba500 10 Cannot find current period zone using local zone 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.694+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.694+0000 7f00a8cba500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.694+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.695+0000 7f00a8cba500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.695+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.695+0000 7f00a8cba500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.695+0000 7f00a8cba500 20 zone default found 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.695+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.695+0000 7f00a8cba500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.695+0000 7f00a8cba500 20 rados->read ofs=0 len=0 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.695+0000 7f00a8cba500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:27.697 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.695+0000 7f00a8cba500 20 started sync module instance, tier type = 2023-09-22T13:15:27.698 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.695+0000 7f00a8cba500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:15:27.704 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.704+0000 7f00a8cba500 20 add_watcher() i=0 2023-09-22T13:15:27.704 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.704+0000 7f00a8cba500 20 add_watcher() i=1 2023-09-22T13:15:27.704 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.704+0000 7f00a8cba500 20 add_watcher() i=2 2023-09-22T13:15:27.704 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.704+0000 7f00a8cba500 20 add_watcher() i=3 2023-09-22T13:15:27.704 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.704+0000 7f00a8cba500 20 add_watcher() i=4 2023-09-22T13:15:27.705 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.704+0000 7f00a8cba500 20 add_watcher() i=5 2023-09-22T13:15:27.705 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.704+0000 7f00a8cba500 20 add_watcher() i=6 2023-09-22T13:15:27.705 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.704+0000 7f009399b700 0 -- 172.21.15.98:0/2641932440 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f008804f340 msgr2=0x7f008804d580 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:15:27.705 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.704+0000 7f00a8cba500 20 add_watcher() i=7 2023-09-22T13:15:27.705 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.704+0000 7f00a8cba500 2 all 8 watchers are set, enabling cache 2023-09-22T13:15:27.706 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.706+0000 7f0065fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:15:27.707 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.706+0000 7f00a8cba500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:15:27.707 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.706+0000 7f00a8cba500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:15:27.711 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.710+0000 7f009499d700 0 -- 172.21.15.98:0/240085632 >> [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] conn(0x5645671f9220 msgr2=0x56456720f8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:15:27.913 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.913+0000 7f009499d700 0 --2- 172.21.15.98:0/240085632 >> [v2:172.21.15.98:3301/0,v1:172.21.15.98:6790/0] conn(0x5645671f9220 0x56456720f8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2023-09-22T13:15:27.915 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.915+0000 7f00a8cba500 5 note: GC not initialized 2023-09-22T13:15:27.916 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.915+0000 7effdcff9700 20 reqs_thread_entry: start 2023-09-22T13:15:27.916 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.915+0000 7effd27fc700 20 reqs_thread_entry: start 2023-09-22T13:15:27.916 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.915+0000 7f00a8cba500 20 init_complete bucket index max shards: 1 2023-09-22T13:15:27.916 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.915+0000 7f00a8cba500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:15:27.917 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.915+0000 7effd0ff9700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:15:58 2023 2023-09-22T13:15:27.917 INFO:tasks.workunit.client.0.smithi098.stderr: 2023-09-22T13:15:27.917 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.915+0000 7effb7fff700 20 reqs_thread_entry: start 2023-09-22T13:15:27.917 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.915+0000 7f00a8cba500 20 get_system_obj_state: rctx=0x7ffde4011d28 obj=default.rgw.meta:users.uid:ceph-test-librgw-file state=0x564567166e80 s->prefetch_data=0 2023-09-22T13:15:27.917 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.915+0000 7f00a8cba500 10 cache get: name=default.rgw.meta+users.uid+ceph-test-librgw-file : miss 2023-09-22T13:15:27.918 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.916+0000 7f00a8cba500 10 cache put: name=default.rgw.meta+users.uid+ceph-test-librgw-file info.flags=0x0 2023-09-22T13:15:27.918 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.916+0000 7f00a8cba500 10 adding default.rgw.meta+users.uid+ceph-test-librgw-file to cache LRU end 2023-09-22T13:15:27.918 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.916+0000 7f00a8cba500 20 get_system_obj_state: rctx=0x7ffde4011d38 obj=default.rgw.meta:users.email:librgw@example.com state=0x564567166e80 s->prefetch_data=0 2023-09-22T13:15:27.918 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.916+0000 7f00a8cba500 10 cache get: name=default.rgw.meta+users.email+librgw@example.com : miss 2023-09-22T13:15:27.918 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.916+0000 7f00a8cba500 10 cache put: name=default.rgw.meta+users.email+librgw@example.com info.flags=0x0 2023-09-22T13:15:27.918 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.916+0000 7f00a8cba500 10 adding default.rgw.meta+users.email+librgw@example.com to cache LRU end 2023-09-22T13:15:27.918 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.916+0000 7f00a8cba500 20 get_system_obj_state: rctx=0x7ffde4011d38 obj=default.rgw.meta:users.keys:net/0lxG+wM8pa5soovTZJOMhF4= state=0x564567166e80 s->prefetch_data=0 2023-09-22T13:15:27.918 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.916+0000 7f00a8cba500 10 cache get: name=default.rgw.meta+users.keys+net/0lxG+wM8pa5soovTZJOMhF4= : miss 2023-09-22T13:15:27.918 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.917+0000 7f00a8cba500 10 cache put: name=default.rgw.meta+users.keys+net/0lxG+wM8pa5soovTZJOMhF4= info.flags=0x0 2023-09-22T13:15:27.918 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.917+0000 7f00a8cba500 10 adding default.rgw.meta+users.keys+net/0lxG+wM8pa5soovTZJOMhF4= to cache LRU end 2023-09-22T13:15:27.919 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.917+0000 7f00a8cba500 20 get_system_obj_state: rctx=0x7ffde4011638 obj=default.rgw.meta:users.keys:net/0lxG+wM8pa5soovTZJOMhF4= state=0x564567166e80 s->prefetch_data=0 2023-09-22T13:15:27.919 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.917+0000 7f00a8cba500 10 cache get: name=default.rgw.meta+users.keys+net/0lxG+wM8pa5soovTZJOMhF4= : hit (negative entry) 2023-09-22T13:15:27.919 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.917+0000 7f00a8cba500 20 get_system_obj_state: rctx=0x7ffde4011978 obj=default.rgw.meta:users.keys:net/0lxG+wM8pa5soovTZJOMhF4= state=0x564567166e80 s->prefetch_data=0 2023-09-22T13:15:27.919 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.917+0000 7f00a8cba500 10 cache get: name=default.rgw.meta+users.keys+net/0lxG+wM8pa5soovTZJOMhF4= : hit (negative entry) 2023-09-22T13:15:27.919 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.918+0000 7f00a8cba500 10 cache put: name=default.rgw.meta+users.uid+ceph-test-librgw-file info.flags=0x17 2023-09-22T13:15:27.920 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.918+0000 7f00a8cba500 10 moving default.rgw.meta+users.uid+ceph-test-librgw-file to cache LRU end 2023-09-22T13:15:27.920 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.918+0000 7f00a8cba500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.uid:ceph-test-librgw-file, ofs0, ns] 2023-09-22T13:15:27.920 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.918+0000 7f0092198700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 992137445377 cookie 94856082637792 notifier 5039 bl.length()=663 2023-09-22T13:15:27.920 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.918+0000 7f0092198700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+ceph-test-librgw-file info.flags=0x17 2023-09-22T13:15:27.920 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.918+0000 7f0092198700 10 rgw watcher librados: moving default.rgw.meta+users.uid+ceph-test-librgw-file to cache LRU end 2023-09-22T13:15:27.920 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.919+0000 7f00a8cba500 10 cache put: name=default.rgw.meta+users.email+librgw@example.com info.flags=0x7 2023-09-22T13:15:27.921 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.919+0000 7f00a8cba500 10 moving default.rgw.meta+users.email+librgw@example.com to cache LRU end 2023-09-22T13:15:27.921 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.919+0000 7f00a8cba500 10 distributing notification oid=default.rgw.control:notify.1 cni=[op: 0, obj: default.rgw.meta:users.email:librgw@example.com, ofs0, ns] 2023-09-22T13:15:27.921 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.920+0000 7f0092198700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 992137445381 cookie 94856082636688 notifier 5039 bl.length()=194 2023-09-22T13:15:27.921 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.920+0000 7f0092198700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.email+librgw@example.com info.flags=0x7 2023-09-22T13:15:27.921 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.920+0000 7f0092198700 10 rgw watcher librados: moving default.rgw.meta+users.email+librgw@example.com to cache LRU end 2023-09-22T13:15:27.922 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.921+0000 7f00a8cba500 10 cache put: name=default.rgw.meta+users.keys+net/0lxG+wM8pa5soovTZJOMhF4= info.flags=0x7 2023-09-22T13:15:27.922 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.921+0000 7f00a8cba500 10 moving default.rgw.meta+users.keys+net/0lxG+wM8pa5soovTZJOMhF4= to cache LRU end 2023-09-22T13:15:27.922 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.921+0000 7f00a8cba500 10 distributing notification oid=default.rgw.control:notify.4 cni=[op: 0, obj: default.rgw.meta:users.keys:net/0lxG+wM8pa5soovTZJOMhF4=, ofs0, ns] 2023-09-22T13:15:27.923 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.922+0000 7f0092198700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 992137445382 cookie 94856082642864 notifier 5039 bl.length()=203 2023-09-22T13:15:27.923 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.922+0000 7f0092198700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+net/0lxG+wM8pa5soovTZJOMhF4= info.flags=0x7 2023-09-22T13:15:27.923 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.922+0000 7f0092198700 10 rgw watcher librados: moving default.rgw.meta+users.keys+net/0lxG+wM8pa5soovTZJOMhF4= to cache LRU end 2023-09-22T13:15:27.923 INFO:tasks.workunit.client.0.smithi098.stdout:{ 2023-09-22T13:15:27.923 INFO:tasks.workunit.client.0.smithi098.stdout: "user_id": "ceph-test-librgw-file", 2023-09-22T13:15:27.923 INFO:tasks.workunit.client.0.smithi098.stdout: "display_name": "librgw test user", 2023-09-22T13:15:27.923 INFO:tasks.workunit.client.0.smithi098.stdout: "email": "librgw@example.com", 2023-09-22T13:15:27.923 INFO:tasks.workunit.client.0.smithi098.stdout: "suspended": 0, 2023-09-22T13:15:27.923 INFO:tasks.workunit.client.0.smithi098.stdout: "max_buckets": 1000, 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: "subusers": [], 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: "keys": [ 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: { 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: "user": "ceph-test-librgw-file", 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: "access_key": "net/0lxG+wM8pa5soovTZJOMhF4=", 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: "secret_key": "jt4h49P64Vy57MsN4Z+1UJCzB52FlkeelupgYjvQLI58JDnYizl3eQ==" 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: } 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: ], 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: "swift_keys": [], 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: "caps": [], 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: "op_mask": "read, write, delete", 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: "default_placement": "", 2023-09-22T13:15:27.924 INFO:tasks.workunit.client.0.smithi098.stdout: "default_storage_class": "", 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "placement_tags": [], 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "bucket_quota": { 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "enabled": false, 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "check_on_raw": false, 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "max_size": -1, 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "max_size_kb": 0, 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "max_objects": -1 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "user_quota": { 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "enabled": false, 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "check_on_raw": false, 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "max_size": -1, 2023-09-22T13:15:27.925 INFO:tasks.workunit.client.0.smithi098.stdout: "max_size_kb": 0, 2023-09-22T13:15:27.926 INFO:tasks.workunit.client.0.smithi098.stdout: "max_objects": -1 2023-09-22T13:15:27.926 INFO:tasks.workunit.client.0.smithi098.stdout: }, 2023-09-22T13:15:27.926 INFO:tasks.workunit.client.0.smithi098.stdout: "temp_url_keys": [], 2023-09-22T13:15:27.926 INFO:tasks.workunit.client.0.smithi098.stdout: "type": "rgw", 2023-09-22T13:15:27.926 INFO:tasks.workunit.client.0.smithi098.stdout: "mfa_ids": [] 2023-09-22T13:15:27.926 INFO:tasks.workunit.client.0.smithi098.stdout:} 2023-09-22T13:15:27.926 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:27.927 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.923+0000 7f00a8cba500 20 remove_watcher() i=0 2023-09-22T13:15:27.927 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.923+0000 7f00a8cba500 2 removed watcher, disabling cache 2023-09-22T13:15:27.927 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.924+0000 7f00a8cba500 20 remove_watcher() i=1 2023-09-22T13:15:27.927 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.925+0000 7f00a8cba500 20 remove_watcher() i=2 2023-09-22T13:15:27.927 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.926+0000 7f00a8cba500 20 remove_watcher() i=3 2023-09-22T13:15:27.928 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.927+0000 7f00a8cba500 20 remove_watcher() i=4 2023-09-22T13:15:27.929 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.928+0000 7f00a8cba500 20 remove_watcher() i=5 2023-09-22T13:15:27.930 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.929+0000 7f00a8cba500 20 remove_watcher() i=6 2023-09-22T13:15:27.931 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.930+0000 7f00a8cba500 20 remove_watcher() i=7 2023-09-22T13:15:27.932 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:27.931+0000 7f0065fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:15:27.936 INFO:tasks.workunit.client.0.smithi098.stdout:phase 1.1 2023-09-22T13:15:27.951 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 24 tests from 1 test suite. 2023-09-22T13:15:27.951 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:27.952 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 24 tests from LibRGW 2023-09-22T13:15:27.952 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:15:27.962 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:27.962 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:28.079 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.INIT (128 ms) 2023-09-22T13:15:28.079 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT_NOROOT 2023-09-22T13:15:28.081 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT_NOROOT (2 ms) 2023-09-22T13:15:28.081 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:15:28.081 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT (0 ms) 2023-09-22T13:15:28.081 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SETUP_HIER1 2023-09-22T13:15:28.088 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/bar/baz/quux 2023-09-22T13:15:28.092 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/f1 2023-09-22T13:15:28.095 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/f2 2023-09-22T13:15:28.098 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/bar/f1 2023-09-22T13:15:28.100 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/bar/d1/ 2023-09-22T13:15:28.103 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/bar/baz/hungry 2023-09-22T13:15:28.106 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/bar/baz/hungry/ 2023-09-22T13:15:28.110 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/bar/baz/momma 2023-09-22T13:15:28.113 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/bar/baz/bear/ 2023-09-22T13:15:28.116 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/bar/baz/sasquatch 2023-09-22T13:15:28.119 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/bar/baz/sasquatch/ 2023-09-22T13:15:28.121 INFO:tasks.workunit.client.0.smithi098.stdout:creating: nfsroot:foo/bar/baz/frobozz 2023-09-22T13:15:28.124 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SETUP_HIER1 (43 ms) 2023-09-22T13:15:28.124 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SETUP_DIRS1 2023-09-22T13:15:28.212 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SETUP_DIRS1 (87 ms) 2023-09-22T13:15:28.212 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SETATTR 2023-09-22T13:15:28.212 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SETATTR (0 ms) 2023-09-22T13:15:28.212 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_CREATE_DIRS1 2023-09-22T13:15:28.228 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_CREATE_DIRS1 (16 ms) 2023-09-22T13:15:28.228 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_SETUP_RENAME1 2023-09-22T13:15:28.261 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_SETUP_RENAME1 (34 ms) 2023-09-22T13:15:28.261 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_INTRABUCKET_RENAME1 2023-09-22T13:15:28.261 INFO:tasks.workunit.client.0.smithi098.stdout:rename file rfile_0 to rfile_r0 (bucket brename0) 2023-09-22T13:15:28.267 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_INTRABUCKET_RENAME1 (6 ms) 2023-09-22T13:15:28.267 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_CROSSBUCKET_RENAME1 2023-09-22T13:15:28.267 INFO:tasks.workunit.client.0.smithi098.stdout:rename file rfile_1 (bucket brename0) to rfile_rhilldog (bucket brename1) 2023-09-22T13:15:28.273 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_CROSSBUCKET_RENAME1 (5 ms) 2023-09-22T13:15:28.273 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.GETATTR_DIRS1 2023-09-22T13:15:28.273 INFO:tasks.workunit.client.0.smithi098.stdout:scanning objects in /bdirs1/dir_0 2023-09-22T13:15:28.273 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_0/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.273 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_0/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:15:28.273 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_0/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.273 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_0/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:15:28.273 INFO:tasks.workunit.client.0.smithi098.stdout:scanning objects in /bdirs1/dir_1 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_1/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_1/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_1/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_1/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout:scanning objects in /bdirs1/dir_2 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_2/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_2/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_2/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_2/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.GETATTR_DIRS1 (0 ms) 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READ_DIRS1 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout:read back objects in /bdirs1/dir_0 2023-09-22T13:15:28.274 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_0/sfile_0 2023-09-22T13:15:28.275 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:15:28.275 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_0/sfile_1 2023-09-22T13:15:28.276 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:15:28.276 INFO:tasks.workunit.client.0.smithi098.stdout:read back objects in /bdirs1/dir_1 2023-09-22T13:15:28.276 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_1/sfile_0 2023-09-22T13:15:28.277 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:15:28.277 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_1/sfile_1 2023-09-22T13:15:28.278 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:15:28.278 INFO:tasks.workunit.client.0.smithi098.stdout:read back objects in /bdirs1/dir_2 2023-09-22T13:15:28.278 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_2/sfile_0 2023-09-22T13:15:28.280 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:15:28.280 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_2/sfile_1 2023-09-22T13:15:28.281 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:15:28.281 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READ_DIRS1 (9 ms) 2023-09-22T13:15:28.281 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READF_SETUP1 2023-09-22T13:15:28.464 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READF_SETUP1 (183 ms) 2023-09-22T13:15:28.465 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READF_DIRS1 2023-09-22T13:15:28.465 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READF_DIRS1 (0 ms) 2023-09-22T13:15:28.465 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.WRITEF_DIRS1 2023-09-22T13:15:28.465 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.WRITEF_DIRS1 (0 ms) 2023-09-22T13:15:28.465 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RELEASE_DIRS1 2023-09-22T13:15:28.465 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:15:28.465 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:15:28.465 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:15:28.465 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:15:28.465 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:15:28.466 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:15:28.466 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:15:28.466 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:15:28.466 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:15:28.466 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:15:28.466 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:15:28.466 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:15:28.466 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RELEASE_DIRS1 (0 ms) 2023-09-22T13:15:28.466 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.HIER1 2023-09-22T13:15:28.472 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/f2 (f2): FILE 2023-09-22T13:15:28.473 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/f1 (f1): FILE 2023-09-22T13:15:28.478 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/f1 (f1): FILE 2023-09-22T13:15:28.481 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/d1 (d1): DIR 2023-09-22T13:15:28.484 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:15:28.485 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:15:28.486 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/quux (quux): FILE 2023-09-22T13:15:28.486 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/momma (momma): FILE 2023-09-22T13:15:28.487 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:15:28.487 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:15:28.488 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/frobozz (frobozz): FILE 2023-09-22T13:15:28.490 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/bear (bear): DIR 2023-09-22T13:15:28.490 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz (baz): DIR 2023-09-22T13:15:28.490 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar (bar): DIR 2023-09-22T13:15:28.490 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo (foo): DIR 2023-09-22T13:15:28.490 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot (nfsroot): DIR 2023-09-22T13:15:28.490 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.HIER1 (25 ms) 2023-09-22T13:15:28.490 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (0 ms) 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:15:28.491 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:15:28.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:28.489+0000 7f1fe6cdd540 -1 shutting down 2023-09-22T13:15:28.492 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:28.489+0000 7f1ef9ffb700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 148517, si_uid : 1000, si_addr0x3e800024425, si_status0 2023-09-22T13:15:28.580 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SHUTDOWN (91 ms) 2023-09-22T13:15:28.581 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 24 tests from LibRGW (629 ms total) 2023-09-22T13:15:28.581 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:28.581 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:28.581 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 24 tests from 1 test suite ran. (629 ms total) 2023-09-22T13:15:28.581 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 24 tests. 2023-09-22T13:15:28.582 INFO:tasks.workunit.client.0.smithi098.stdout:phase 1.2 2023-09-22T13:15:28.598 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 24 tests from 1 test suite. 2023-09-22T13:15:28.598 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:28.598 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 24 tests from LibRGW 2023-09-22T13:15:28.598 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:15:28.606 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:28.606 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:28.692 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.INIT (95 ms) 2023-09-22T13:15:28.692 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT_NOROOT 2023-09-22T13:15:28.694 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT_NOROOT (1 ms) 2023-09-22T13:15:28.694 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:15:28.694 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT (0 ms) 2023-09-22T13:15:28.694 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SETUP_HIER1 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SETUP_HIER1 (0 ms) 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SETUP_DIRS1 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SETUP_DIRS1 (0 ms) 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SETATTR 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SETATTR (0 ms) 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_CREATE_DIRS1 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_CREATE_DIRS1 (0 ms) 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_SETUP_RENAME1 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_SETUP_RENAME1 (0 ms) 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_INTRABUCKET_RENAME1 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_INTRABUCKET_RENAME1 (0 ms) 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_CROSSBUCKET_RENAME1 2023-09-22T13:15:28.695 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_CROSSBUCKET_RENAME1 (0 ms) 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.GETATTR_DIRS1 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.GETATTR_DIRS1 (0 ms) 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READ_DIRS1 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READ_DIRS1 (0 ms) 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READF_SETUP1 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READF_SETUP1 (0 ms) 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READF_DIRS1 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READF_DIRS1 (0 ms) 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.WRITEF_DIRS1 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.WRITEF_DIRS1 (0 ms) 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RELEASE_DIRS1 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RELEASE_DIRS1 (0 ms) 2023-09-22T13:15:28.696 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.HIER1 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.HIER1 (0 ms) 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (0 ms) 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:15:28.697 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:15:28.698 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:15:28.698 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:28.693+0000 7f6e19692540 -1 shutting down 2023-09-22T13:15:28.698 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:28.693+0000 7f6d2d7fa700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 148601, si_uid : 1000, si_addr0x3e800024479, si_status0 2023-09-22T13:15:28.793 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SHUTDOWN (100 ms) 2023-09-22T13:15:28.793 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 24 tests from LibRGW (196 ms total) 2023-09-22T13:15:28.793 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:28.793 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:28.793 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 24 tests from 1 test suite ran. (196 ms total) 2023-09-22T13:15:28.793 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 24 tests. 2023-09-22T13:15:28.795 INFO:tasks.workunit.client.0.smithi098.stdout:phase 1.3 2023-09-22T13:15:28.810 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 24 tests from 1 test suite. 2023-09-22T13:15:28.810 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:28.811 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 24 tests from LibRGW 2023-09-22T13:15:28.811 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:15:28.820 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:28.821 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:28.897 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.INIT (87 ms) 2023-09-22T13:15:28.897 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT_NOROOT 2023-09-22T13:15:28.899 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT_NOROOT (1 ms) 2023-09-22T13:15:28.899 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:15:28.899 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT (0 ms) 2023-09-22T13:15:28.899 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SETUP_HIER1 2023-09-22T13:15:28.903 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SETUP_HIER1 (5 ms) 2023-09-22T13:15:28.903 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SETUP_DIRS1 2023-09-22T13:15:28.963 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SETUP_DIRS1 (60 ms) 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SETATTR 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SETATTR (0 ms) 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_CREATE_DIRS1 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_CREATE_DIRS1 (0 ms) 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_SETUP_RENAME1 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_SETUP_RENAME1 (0 ms) 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_INTRABUCKET_RENAME1 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_INTRABUCKET_RENAME1 (0 ms) 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RGW_CROSSBUCKET_RENAME1 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RGW_CROSSBUCKET_RENAME1 (0 ms) 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.GETATTR_DIRS1 2023-09-22T13:15:28.964 INFO:tasks.workunit.client.0.smithi098.stdout:scanning objects in /bdirs1/dir_0 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_0/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_0/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_0/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_0/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout:scanning objects in /bdirs1/dir_1 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_1/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_1/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_1/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_1/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout:scanning objects in /bdirs1/dir_2 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_2/sdir_0 (sdir_0): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.965 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_2/sfile_0 (sfile_0): FILE mode: 33412 size: 16 2023-09-22T13:15:28.966 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_2/sdir_1 (sdir_1): DIR mode: 17139 nlinks: 1 2023-09-22T13:15:28.966 INFO:tasks.workunit.client.0.smithi098.stdout: /bdirs1/dir_2/sfile_1 (sfile_1): FILE mode: 33412 size: 16 2023-09-22T13:15:28.966 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.GETATTR_DIRS1 (0 ms) 2023-09-22T13:15:28.966 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READ_DIRS1 2023-09-22T13:15:28.966 INFO:tasks.workunit.client.0.smithi098.stdout:read back objects in /bdirs1/dir_0 2023-09-22T13:15:28.966 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_0/sfile_0 2023-09-22T13:15:28.966 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:15:28.966 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_0/sfile_1 2023-09-22T13:15:28.967 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:15:28.967 INFO:tasks.workunit.client.0.smithi098.stdout:read back objects in /bdirs1/dir_1 2023-09-22T13:15:28.967 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_1/sfile_0 2023-09-22T13:15:28.968 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:15:28.968 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_1/sfile_1 2023-09-22T13:15:28.970 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:15:28.970 INFO:tasks.workunit.client.0.smithi098.stdout:read back objects in /bdirs1/dir_2 2023-09-22T13:15:28.970 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_2/sfile_0 2023-09-22T13:15:28.971 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_0 : "data for sfile_0" 2023-09-22T13:15:28.972 INFO:tasks.workunit.client.0.smithi098.stdout:reading 0,256 dir_2/sfile_1 2023-09-22T13:15:28.973 INFO:tasks.workunit.client.0.smithi098.stdout: read back from sfile_1 : "data for sfile_1" 2023-09-22T13:15:28.974 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READ_DIRS1 (10 ms) 2023-09-22T13:15:28.974 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READF_SETUP1 2023-09-22T13:15:28.974 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READF_SETUP1 (0 ms) 2023-09-22T13:15:28.974 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READF_DIRS1 2023-09-22T13:15:28.974 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READF_DIRS1 (0 ms) 2023-09-22T13:15:28.974 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.WRITEF_DIRS1 2023-09-22T13:15:28.974 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.WRITEF_DIRS1 (0 ms) 2023-09-22T13:15:28.974 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.RELEASE_DIRS1 2023-09-22T13:15:28.974 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:15:28.974 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_0 type: DIR refs: 2 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_0 type: FILE refs: 2 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:release sdir_1 type: DIR refs: 2 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:release sfile_1 type: FILE refs: 2 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.RELEASE_DIRS1 (0 ms) 2023-09-22T13:15:28.975 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.HIER1 2023-09-22T13:15:28.979 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/f2 (f2): FILE 2023-09-22T13:15:28.979 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/f1 (f1): FILE 2023-09-22T13:15:28.982 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/f1 (f1): FILE 2023-09-22T13:15:28.983 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/d1 (d1): DIR 2023-09-22T13:15:28.986 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:15:28.986 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/sasquatch (sasquatch): FILE 2023-09-22T13:15:28.987 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/quux (quux): FILE 2023-09-22T13:15:28.988 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/momma (momma): FILE 2023-09-22T13:15:28.989 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:15:28.989 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/hungry (hungry): FILE 2023-09-22T13:15:28.990 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/frobozz (frobozz): FILE 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz/bear (bear): DIR 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar/baz (baz): DIR 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo/bar (bar): DIR 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot/foo (foo): DIR 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:/nfsroot (nfsroot): DIR 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.HIER1 (18 ms) 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (0 ms) 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:15:28.992 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:15:28.993 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:15:28.993 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:15:28.993 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:15:28.993 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:15:28.993 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:15:28.993 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:15:28.993 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:15:28.993 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:28.992+0000 7fc03f90f540 -1 shutting down 2023-09-22T13:15:28.993 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:28.992+0000 7fbf46ffd700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 148685, si_uid : 1000, si_addr0x3e8000244cd, si_status0 2023-09-22T13:15:29.098 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SHUTDOWN (107 ms) 2023-09-22T13:15:29.098 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 24 tests from LibRGW (288 ms total) 2023-09-22T13:15:29.099 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:29.099 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:29.099 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 24 tests from 1 test suite ran. (288 ms total) 2023-09-22T13:15:29.099 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 24 tests. 2023-09-22T13:15:29.100 INFO:tasks.workunit.client.0.smithi098.stdout:phase 2.1 2023-09-22T13:15:29.115 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 9 tests from 1 test suite. 2023-09-22T13:15:29.116 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:29.116 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 9 tests from LibRGW 2023-09-22T13:15:29.116 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:15:29.124 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:29.124 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:29.213 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.INIT (98 ms) 2023-09-22T13:15:29.213 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:15:29.214 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:15:29.215 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:15:29.220 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CREATE_BUCKET (6 ms) 2023-09-22T13:15:29.220 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:15:29.220 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_BUCKET (0 ms) 2023-09-22T13:15:29.221 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CREATE_BUCKET_MULTI 2023-09-22T13:15:29.226 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave0 2023-09-22T13:15:29.232 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave1 2023-09-22T13:15:29.237 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave2 2023-09-22T13:15:29.243 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave3 2023-09-22T13:15:29.249 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave4 2023-09-22T13:15:29.255 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave5 2023-09-22T13:15:29.261 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave6 2023-09-22T13:15:29.267 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave7 2023-09-22T13:15:29.273 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave8 2023-09-22T13:15:29.278 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave9 2023-09-22T13:15:29.278 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CREATE_BUCKET_MULTI (58 ms) 2023-09-22T13:15:29.279 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_BUCKET_MULTI 2023-09-22T13:15:29.451 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_BUCKET_MULTI (173 ms) 2023-09-22T13:15:29.451 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:15:29.451 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:15:29.451 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:15:29.452 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:15:29.452 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:15:29.452 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:29.451+0000 7f47a20b5540 -1 shutting down 2023-09-22T13:15:29.452 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:29.451+0000 7f46b5ffb700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 148769, si_uid : 1000, si_addr0x3e800024521, si_status0 2023-09-22T13:15:29.514 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SHUTDOWN (63 ms) 2023-09-22T13:15:29.549 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 9 tests from LibRGW (399 ms total) 2023-09-22T13:15:29.549 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:29.549 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:29.549 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 9 tests from 1 test suite ran. (399 ms total) 2023-09-22T13:15:29.549 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 9 tests. 2023-09-22T13:15:29.550 INFO:tasks.workunit.client.0.smithi098.stdout:phase 2.2 2023-09-22T13:15:29.550 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 9 tests from 1 test suite. 2023-09-22T13:15:29.550 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:29.550 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 9 tests from LibRGW 2023-09-22T13:15:29.550 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:15:29.551 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:29.551 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:29.626 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.INIT (95 ms) 2023-09-22T13:15:29.626 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:15:29.628 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT (2 ms) 2023-09-22T13:15:29.628 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:15:29.628 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CREATE_BUCKET (0 ms) 2023-09-22T13:15:29.628 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:15:29.649 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_BUCKET (20 ms) 2023-09-22T13:15:29.649 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CREATE_BUCKET_MULTI 2023-09-22T13:15:29.654 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave0 2023-09-22T13:15:29.660 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave1 2023-09-22T13:15:29.666 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave2 2023-09-22T13:15:29.671 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave3 2023-09-22T13:15:29.677 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave4 2023-09-22T13:15:29.683 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave5 2023-09-22T13:15:29.688 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave6 2023-09-22T13:15:29.694 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave7 2023-09-22T13:15:29.700 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave8 2023-09-22T13:15:29.706 INFO:tasks.workunit.client.0.smithi098.stdout:created: sorrydave9 2023-09-22T13:15:29.706 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CREATE_BUCKET_MULTI (58 ms) 2023-09-22T13:15:29.706 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_BUCKET_MULTI 2023-09-22T13:15:29.875 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_BUCKET_MULTI (169 ms) 2023-09-22T13:15:29.875 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:15:29.875 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:15:29.875 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:15:29.876 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:15:29.876 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:15:29.876 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:29.875+0000 7f171ba0a540 -1 shutting down 2023-09-22T13:15:29.876 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:29.875+0000 7f16237fe700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 148853, si_uid : 1000, si_addr0x3e800024575, si_status0 2023-09-22T13:15:29.927 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SHUTDOWN (52 ms) 2023-09-22T13:15:29.927 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 9 tests from LibRGW (396 ms total) 2023-09-22T13:15:29.928 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:29.928 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:29.928 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 9 tests from 1 test suite ran. (396 ms total) 2023-09-22T13:15:29.928 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 9 tests. 2023-09-22T13:15:29.929 INFO:tasks.workunit.client.0.smithi098.stdout:phase 3.1 2023-09-22T13:15:30.692 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 13 tests from 1 test suite. 2023-09-22T13:15:30.692 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:30.692 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 13 tests from LibRGW 2023-09-22T13:15:30.692 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:15:30.701 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:30.701 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:30.788 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.INIT (97 ms) 2023-09-22T13:15:30.788 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:15:30.789 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:15:30.790 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:15:30.795 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CREATE_BUCKET (6 ms) 2023-09-22T13:15:30.795 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:15:30.798 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (3 ms) 2023-09-22T13:15:30.798 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:15:30.800 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (2 ms) 2023-09-22T13:15:30.800 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LARGE1 2023-09-22T13:15:30.878 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LARGE1 (78 ms) 2023-09-22T13:15:30.878 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LARGE2 2023-09-22T13:15:30.957 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LARGE2 (79 ms) 2023-09-22T13:15:30.957 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:15:30.958 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:15:30.958 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:15:30.958 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_OBJECT (0 ms) 2023-09-22T13:15:30.958 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:15:30.958 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_BUCKET (0 ms) 2023-09-22T13:15:30.958 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:15:30.958 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:15:30.958 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:15:30.958 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:15:30.958 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:15:30.959 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:30.957+0000 7fabc498a540 -1 shutting down 2023-09-22T13:15:30.959 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:30.958+0000 7faad9ffb700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 148937, si_uid : 1000, si_addr0x3e8000245c9, si_status0 2023-09-22T13:15:30.990 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SHUTDOWN (32 ms) 2023-09-22T13:15:30.990 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 13 tests from LibRGW (298 ms total) 2023-09-22T13:15:30.990 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:30.990 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:30.990 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 13 tests from 1 test suite ran. (298 ms total) 2023-09-22T13:15:30.990 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 13 tests. 2023-09-22T13:15:30.992 INFO:tasks.workunit.client.0.smithi098.stdout:phase 3.2 2023-09-22T13:15:31.746 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 13 tests from 1 test suite. 2023-09-22T13:15:31.747 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:31.747 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 13 tests from LibRGW 2023-09-22T13:15:31.748 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:15:31.754 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:31.754 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:32.032 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.INIT (286 ms) 2023-09-22T13:15:32.032 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:15:32.033 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:15:32.034 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:15:32.034 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CREATE_BUCKET (0 ms) 2023-09-22T13:15:32.034 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:15:32.038 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (5 ms) 2023-09-22T13:15:32.039 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:15:32.043 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (4 ms) 2023-09-22T13:15:32.043 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LARGE1 2023-09-22T13:15:32.112 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LARGE1 (69 ms) 2023-09-22T13:15:32.112 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LARGE2 2023-09-22T13:15:32.112 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LARGE2 (0 ms) 2023-09-22T13:15:32.112 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:15:32.112 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:15:32.112 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:15:32.116 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_OBJECT (4 ms) 2023-09-22T13:15:32.116 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:15:32.134 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_BUCKET (19 ms) 2023-09-22T13:15:32.134 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:15:32.135 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:15:32.135 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:15:32.135 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:15:32.135 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:15:32.135 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:32.134+0000 7fd0c216d540 -1 shutting down 2023-09-22T13:15:32.135 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:32.135+0000 7fcfc77fe700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 149021, si_uid : 1000, si_addr0x3e80002461d, si_status0 2023-09-22T13:15:32.233 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SHUTDOWN (99 ms) 2023-09-22T13:15:32.233 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 13 tests from LibRGW (487 ms total) 2023-09-22T13:15:32.233 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:32.233 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:32.234 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 13 tests from 1 test suite ran. (487 ms total) 2023-09-22T13:15:32.234 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 13 tests. 2023-09-22T13:15:32.235 INFO:tasks.workunit.client.0.smithi098.stdout:phase 4.1 2023-09-22T13:15:32.251 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 11 tests from 1 test suite. 2023-09-22T13:15:32.251 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:32.251 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 11 tests from LibRGW 2023-09-22T13:15:32.252 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.TVAR 2023-09-22T13:15:32.252 INFO:tasks.workunit.client.0.smithi098.stdout:read i1: 64001 2023-09-22T13:15:32.252 INFO:tasks.workunit.client.0.smithi098.stdout:read s1: blunderbuss 2023-09-22T13:15:32.252 INFO:tasks.workunit.client.0.smithi098.stdout:read s3: [ OK ] LibRGW.TVAR (0 ms) 2023-09-22T13:15:32.252 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:15:32.262 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:32.262 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:32.340 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.INIT (88 ms) 2023-09-22T13:15:32.340 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:15:32.341 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT (2 ms) 2023-09-22T13:15:32.341 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:15:32.354 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (13 ms) 2023-09-22T13:15:32.354 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:15:32.755 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_0 2023-09-22T13:15:32.756 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_1 2023-09-22T13:15:32.756 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_2 2023-09-22T13:15:32.756 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_3 2023-09-22T13:15:32.756 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_4 2023-09-22T13:15:32.756 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_5 2023-09-22T13:15:32.756 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_6 2023-09-22T13:15:32.756 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_7 2023-09-22T13:15:32.756 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_8 2023-09-22T13:15:32.756 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_9 2023-09-22T13:15:32.756 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_10 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_11 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_12 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_13 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_14 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_15 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_16 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_17 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_18 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_19 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_20 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_21 2023-09-22T13:15:32.757 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_22 2023-09-22T13:15:32.758 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_23 2023-09-22T13:15:32.758 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_24 2023-09-22T13:15:32.758 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_25 2023-09-22T13:15:32.758 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_26 2023-09-22T13:15:32.758 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_27 2023-09-22T13:15:32.758 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_28 2023-09-22T13:15:32.758 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_29 2023-09-22T13:15:32.758 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_30 2023-09-22T13:15:32.758 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_31 2023-09-22T13:15:32.758 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_32 2023-09-22T13:15:32.758 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_33 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_34 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_35 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_36 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_37 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_38 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_39 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_40 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_41 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_42 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_43 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_44 2023-09-22T13:15:32.759 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_45 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_46 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_47 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_48 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_49 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_50 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_51 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_52 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_53 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_54 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_55 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_56 2023-09-22T13:15:32.760 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_57 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_58 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_59 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_60 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_61 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_62 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_63 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_64 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_65 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_66 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_67 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_68 2023-09-22T13:15:32.761 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_69 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_70 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_71 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_72 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_73 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_74 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_75 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_76 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_77 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_78 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_79 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_80 2023-09-22T13:15:32.762 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_81 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_82 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_83 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_84 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_85 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_86 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_87 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_88 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_89 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_90 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_91 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_92 2023-09-22T13:15:32.763 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_93 2023-09-22T13:15:32.764 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_94 2023-09-22T13:15:32.764 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_95 2023-09-22T13:15:32.764 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_96 2023-09-22T13:15:32.764 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_97 2023-09-22T13:15:32.764 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_98 2023-09-22T13:15:32.764 INFO:tasks.workunit.client.0.smithi098.stdout:created: dmarker:f_99 2023-09-22T13:15:32.764 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (401 ms) 2023-09-22T13:15:32.764 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:15:32.765 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=. flags=4 2023-09-22T13:15:32.765 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=.. flags=4 2023-09-22T13:15:32.765 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_0 flags=8 2023-09-22T13:15:32.765 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_1 flags=8 2023-09-22T13:15:32.766 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_10 flags=8 2023-09-22T13:15:32.766 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_11 flags=8 2023-09-22T13:15:32.766 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_12 flags=8 2023-09-22T13:15:32.766 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_13 flags=8 2023-09-22T13:15:32.766 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_14 flags=8 2023-09-22T13:15:32.766 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_15 flags=8 2023-09-22T13:15:32.766 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_16 flags=8 2023-09-22T13:15:32.766 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_17 flags=8 2023-09-22T13:15:32.766 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_18 flags=8 2023-09-22T13:15:32.766 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_19 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_2 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_20 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_21 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_22 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_23 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_24 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_25 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_26 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_27 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_28 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_29 flags=8 2023-09-22T13:15:32.767 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_3 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_30 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_31 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_32 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_33 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_34 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_35 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_36 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_37 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_38 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_39 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_4 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_40 flags=8 2023-09-22T13:15:32.768 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_41 flags=8 2023-09-22T13:15:32.769 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_42 flags=8 2023-09-22T13:15:32.769 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_43 flags=8 2023-09-22T13:15:32.769 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_44 flags=8 2023-09-22T13:15:32.769 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_45 flags=8 2023-09-22T13:15:32.769 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_46 flags=8 2023-09-22T13:15:32.769 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_47 flags=8 2023-09-22T13:15:32.769 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_48 flags=8 2023-09-22T13:15:32.769 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_49 flags=8 2023-09-22T13:15:32.769 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_5 flags=8 2023-09-22T13:15:32.769 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_50 flags=8 2023-09-22T13:15:32.769 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_51 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_52 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_53 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_54 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_55 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_56 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_57 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_58 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_59 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_6 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_60 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_61 flags=8 2023-09-22T13:15:32.770 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_62 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_63 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_64 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_65 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_66 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_67 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_68 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_69 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_7 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_70 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_71 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_72 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_73 flags=8 2023-09-22T13:15:32.771 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_74 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_75 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_76 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_77 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_78 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_79 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_8 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_80 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_81 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_82 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_83 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_84 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_85 flags=8 2023-09-22T13:15:32.772 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_86 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_87 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_88 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_89 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_9 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_90 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_91 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_92 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_93 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_94 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_95 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_96 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_97 flags=8 2023-09-22T13:15:32.773 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_98 flags=8 2023-09-22T13:15:32.774 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_99 flags=8 2023-09-22T13:15:32.774 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_READDIR (3 ms) 2023-09-22T13:15:32.774 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER2_READDIR 2023-09-22T13:15:32.774 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_0 flags=8 2023-09-22T13:15:32.774 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_1 flags=8 2023-09-22T13:15:32.774 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_10 flags=8 2023-09-22T13:15:32.774 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_11 flags=8 2023-09-22T13:15:32.774 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_12 flags=8 2023-09-22T13:15:32.774 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_13 flags=8 2023-09-22T13:15:32.774 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_14 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_15 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_16 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_17 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_18 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_19 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_2 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_20 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_21 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_22 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_23 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_24 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_25 flags=8 2023-09-22T13:15:32.775 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_26 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_27 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_28 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_29 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_3 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_30 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_31 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_32 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_33 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_34 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_35 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_36 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_37 flags=8 2023-09-22T13:15:32.776 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_38 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_39 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_4 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_40 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_41 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_42 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_43 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_44 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_45 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_46 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_47 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_48 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_49 flags=8 2023-09-22T13:15:32.777 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_5 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_50 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_51 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_52 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_53 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_54 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_55 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_56 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_57 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_58 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_59 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_6 flags=8 2023-09-22T13:15:32.778 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_60 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_61 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_62 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_63 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_64 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_65 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_66 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_67 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_68 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_69 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_7 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_70 flags=8 2023-09-22T13:15:32.779 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_71 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_72 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_73 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_74 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_75 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_76 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_77 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_78 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_79 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_8 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_80 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_81 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_82 flags=8 2023-09-22T13:15:32.780 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_83 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_84 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_85 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_86 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_87 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_88 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_89 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_9 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_90 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_91 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_92 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_93 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_94 flags=8 2023-09-22T13:15:32.781 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_95 flags=8 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_96 flags=8 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_97 flags=8 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_98 flags=8 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_99 flags=8 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER2_READDIR (2 ms) 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:15:32.782 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:15:32.783 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:32.760+0000 7f23a1aa5540 -1 shutting down 2023-09-22T13:15:32.783 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:32.761+0000 7f22b9ffb700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 149105, si_uid : 1000, si_addr0x3e800024671, si_status0 2023-09-22T13:15:32.841 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SHUTDOWN (81 ms) 2023-09-22T13:15:32.848 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 11 tests from LibRGW (590 ms total) 2023-09-22T13:15:32.848 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:32.848 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:32.849 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 11 tests from 1 test suite ran. (590 ms total) 2023-09-22T13:15:32.849 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 11 tests. 2023-09-22T13:15:32.849 INFO:tasks.workunit.client.0.smithi098.stdout:phase 4.2 2023-09-22T13:15:32.858 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 11 tests from 1 test suite. 2023-09-22T13:15:32.858 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:32.858 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 11 tests from LibRGW 2023-09-22T13:15:32.858 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.TVAR 2023-09-22T13:15:32.859 INFO:tasks.workunit.client.0.smithi098.stdout:read i1: 64001 2023-09-22T13:15:32.859 INFO:tasks.workunit.client.0.smithi098.stdout:read s1: blunderbuss 2023-09-22T13:15:32.859 INFO:tasks.workunit.client.0.smithi098.stdout:read s3: [ OK ] LibRGW.TVAR (0 ms) 2023-09-22T13:15:32.859 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:15:32.866 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:32.866 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:32.942 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.INIT (83 ms) 2023-09-22T13:15:32.943 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:15:32.943 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT (1 ms) 2023-09-22T13:15:32.943 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_SETUP_BUCKET 2023-09-22T13:15:32.952 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_SETUP_BUCKET (9 ms) 2023-09-22T13:15:32.952 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_SETUP_OBJECTS 2023-09-22T13:15:32.952 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_SETUP_OBJECTS (0 ms) 2023-09-22T13:15:32.952 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_READDIR 2023-09-22T13:15:32.952 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_READDIR (0 ms) 2023-09-22T13:15:32.952 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER2_READDIR 2023-09-22T13:15:32.954 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_0 flags=8 2023-09-22T13:15:32.954 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_1 flags=8 2023-09-22T13:15:32.954 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_10 flags=8 2023-09-22T13:15:32.955 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_11 flags=8 2023-09-22T13:15:32.955 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_12 flags=8 2023-09-22T13:15:32.955 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_13 flags=8 2023-09-22T13:15:32.955 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_14 flags=8 2023-09-22T13:15:32.955 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_15 flags=8 2023-09-22T13:15:32.955 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_16 flags=8 2023-09-22T13:15:32.955 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_17 flags=8 2023-09-22T13:15:32.955 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_18 flags=8 2023-09-22T13:15:32.955 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_19 flags=8 2023-09-22T13:15:32.955 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_2 flags=8 2023-09-22T13:15:32.955 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_20 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_21 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_22 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_23 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_24 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_25 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_26 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_27 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_28 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_29 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_3 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_30 flags=8 2023-09-22T13:15:32.956 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_31 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_32 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_33 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_34 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_35 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_36 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_37 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_38 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_39 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_4 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_40 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_41 flags=8 2023-09-22T13:15:32.957 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_42 flags=8 2023-09-22T13:15:32.958 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_43 flags=8 2023-09-22T13:15:32.958 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_44 flags=8 2023-09-22T13:15:32.958 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_45 flags=8 2023-09-22T13:15:32.958 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_46 flags=8 2023-09-22T13:15:32.958 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_47 flags=8 2023-09-22T13:15:32.958 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_48 flags=8 2023-09-22T13:15:32.958 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_49 flags=8 2023-09-22T13:15:32.958 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_5 flags=8 2023-09-22T13:15:32.958 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_50 flags=8 2023-09-22T13:15:32.958 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_51 flags=8 2023-09-22T13:15:32.958 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_52 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_53 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_54 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_55 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_56 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_57 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_58 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_59 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_6 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_60 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_61 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_62 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_63 flags=8 2023-09-22T13:15:32.959 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_64 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_65 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_66 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_67 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_68 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_69 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_7 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_70 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_71 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_72 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_73 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_74 flags=8 2023-09-22T13:15:32.960 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_75 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_76 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_77 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_78 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_79 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_8 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_80 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_81 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_82 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_83 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_84 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_85 flags=8 2023-09-22T13:15:32.961 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_86 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_87 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_88 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_89 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_9 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_90 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_91 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_92 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_93 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_94 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_95 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_96 flags=8 2023-09-22T13:15:32.962 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_97 flags=8 2023-09-22T13:15:32.963 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_98 flags=8 2023-09-22T13:15:32.963 INFO:tasks.workunit.client.0.smithi098.stdout:r2_cb bucket=dmarker dir=nfs_marker iv count=0 called back name=f_99 flags=8 2023-09-22T13:15:32.963 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER2_READDIR (3 ms) 2023-09-22T13:15:32.963 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MARKER1_OBJ_CLEANUP 2023-09-22T13:15:32.963 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MARKER1_OBJ_CLEANUP (0 ms) 2023-09-22T13:15:32.963 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:15:32.963 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:15:32.963 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:15:32.963 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:15:32.963 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:15:32.963 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:32.954+0000 7fe8df802540 -1 shutting down 2023-09-22T13:15:32.964 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:32.954+0000 7fe7e6ffd700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 149189, si_uid : 1000, si_addr0x3e8000246c5, si_status0 2023-09-22T13:15:33.042 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SHUTDOWN (88 ms) 2023-09-22T13:15:33.043 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 11 tests from LibRGW (184 ms total) 2023-09-22T13:15:33.043 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:33.043 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:33.043 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 11 tests from 1 test suite ran. (184 ms total) 2023-09-22T13:15:33.043 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 11 tests. 2023-09-22T13:15:33.044 INFO:tasks.workunit.client.0.smithi098.stdout:phase 5.1 2023-09-22T13:15:33.084 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 19 tests from 1 test suite. 2023-09-22T13:15:33.084 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:33.084 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 19 tests from LibRGW 2023-09-22T13:15:33.084 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:15:33.094 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:33.094 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:33.180 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.INIT (97 ms) 2023-09-22T13:15:33.180 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:15:33.182 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT (2 ms) 2023-09-22T13:15:33.182 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:15:33.188 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CREATE_BUCKET (6 ms) 2023-09-22T13:15:33.188 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:15:33.191 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (3 ms) 2023-09-22T13:15:33.191 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LIST_OBJECTS 2023-09-22T13:15:33.191 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LIST_OBJECTS (0 ms) 2023-09-22T13:15:33.191 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:15:33.193 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (2 ms) 2023-09-22T13:15:33.193 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.OBJ_OPEN 2023-09-22T13:15:33.193 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.OBJ_OPEN (0 ms) 2023-09-22T13:15:33.194 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.PUT_OBJECT 2023-09-22T13:15:33.197 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.PUT_OBJECT (4 ms) 2023-09-22T13:15:33.197 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.GET_OBJECT 2023-09-22T13:15:33.198 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.GET_OBJECT (1 ms) 2023-09-22T13:15:33.198 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:15:33.199 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:15:33.199 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.WRITE_READ_VERIFY 2023-09-22T13:15:33.199 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.WRITE_READ_VERIFY (0 ms) 2023-09-22T13:15:33.199 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.WRITEV 2023-09-22T13:15:33.199 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.WRITEV (0 ms) 2023-09-22T13:15:33.199 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READV 2023-09-22T13:15:33.199 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READV (0 ms) 2023-09-22T13:15:33.199 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READV_AFTER_WRITEV 2023-09-22T13:15:33.199 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READV_AFTER_WRITEV (0 ms) 2023-09-22T13:15:33.199 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:15:33.199 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_OBJECT (0 ms) 2023-09-22T13:15:33.200 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:15:33.200 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_BUCKET (0 ms) 2023-09-22T13:15:33.200 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:15:33.200 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:15:33.200 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:15:33.200 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:15:33.200 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:15:33.200 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:33.198+0000 7fba63884540 -1 shutting down 2023-09-22T13:15:33.200 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:33.198+0000 7fb96affd700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 149273, si_uid : 1000, si_addr0x3e800024719, si_status0 2023-09-22T13:15:33.281 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SHUTDOWN (83 ms) 2023-09-22T13:15:33.281 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 19 tests from LibRGW (198 ms total) 2023-09-22T13:15:33.281 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:33.281 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:33.282 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 19 tests from 1 test suite ran. (198 ms total) 2023-09-22T13:15:33.282 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 19 tests. 2023-09-22T13:15:33.283 INFO:tasks.workunit.client.0.smithi098.stdout:phase 5.2 2023-09-22T13:15:33.322 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] Running 19 tests from 1 test suite. 2023-09-22T13:15:33.322 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment set-up. 2023-09-22T13:15:33.322 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 19 tests from LibRGW 2023-09-22T13:15:33.323 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.INIT 2023-09-22T13:15:33.329 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:33.329 INFO:tasks.workunit.client.0.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:33.404 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.INIT (81 ms) 2023-09-22T13:15:33.404 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.MOUNT 2023-09-22T13:15:33.405 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.MOUNT (2 ms) 2023-09-22T13:15:33.405 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CREATE_BUCKET 2023-09-22T13:15:33.405 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CREATE_BUCKET (0 ms) 2023-09-22T13:15:33.405 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LOOKUP_BUCKET 2023-09-22T13:15:33.410 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LOOKUP_BUCKET (5 ms) 2023-09-22T13:15:33.411 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LIST_OBJECTS 2023-09-22T13:15:33.411 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LIST_OBJECTS (0 ms) 2023-09-22T13:15:33.411 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.LOOKUP_OBJECT 2023-09-22T13:15:33.411 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.LOOKUP_OBJECT (0 ms) 2023-09-22T13:15:33.411 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.OBJ_OPEN 2023-09-22T13:15:33.411 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.OBJ_OPEN (0 ms) 2023-09-22T13:15:33.411 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.PUT_OBJECT 2023-09-22T13:15:33.411 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.PUT_OBJECT (0 ms) 2023-09-22T13:15:33.411 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.GET_OBJECT 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.GET_OBJECT (0 ms) 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.STAT_OBJECT 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.STAT_OBJECT (0 ms) 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.WRITE_READ_VERIFY 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.WRITE_READ_VERIFY (0 ms) 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.WRITEV 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.WRITEV (0 ms) 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READV 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READV (0 ms) 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.READV_AFTER_WRITEV 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.READV_AFTER_WRITEV (0 ms) 2023-09-22T13:15:33.412 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_OBJECT 2023-09-22T13:15:33.414 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_OBJECT (4 ms) 2023-09-22T13:15:33.414 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.DELETE_BUCKET 2023-09-22T13:15:33.433 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.DELETE_BUCKET (19 ms) 2023-09-22T13:15:33.434 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.CLEANUP 2023-09-22T13:15:33.434 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.CLEANUP (0 ms) 2023-09-22T13:15:33.434 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.UMOUNT 2023-09-22T13:15:33.434 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.UMOUNT (0 ms) 2023-09-22T13:15:33.434 INFO:tasks.workunit.client.0.smithi098.stdout:[ RUN ] LibRGW.SHUTDOWN 2023-09-22T13:15:33.434 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:33.434+0000 7f443f0fc540 -1 shutting down 2023-09-22T13:15:33.435 INFO:tasks.workunit.client.0.smithi098.stderr:2023-09-22T13:15:33.434+0000 7f4355ffb700 -1 received signal: User defined signal 1, si_code : -6, si_value (int): 0, si_value (ptr): 0, si_errno: 0, si_pid : 149357, si_uid : 1000, si_addr0x3e80002476d, si_status0 2023-09-22T13:15:33.505 INFO:tasks.workunit.client.0.smithi098.stdout:[ OK ] LibRGW.SHUTDOWN (71 ms) 2023-09-22T13:15:33.505 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] 19 tests from LibRGW (182 ms total) 2023-09-22T13:15:33.505 INFO:tasks.workunit.client.0.smithi098.stdout: 2023-09-22T13:15:33.505 INFO:tasks.workunit.client.0.smithi098.stdout:[----------] Global test environment tear-down 2023-09-22T13:15:33.505 INFO:tasks.workunit.client.0.smithi098.stdout:[==========] 19 tests from 1 test suite ran. (182 ms total) 2023-09-22T13:15:33.505 INFO:tasks.workunit.client.0.smithi098.stdout:[ PASSED ] 19 tests. 2023-09-22T13:15:33.507 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-22T13:15:33.507 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-22T13:15:33.535 INFO:tasks.workunit:Stopping ['cls/test_cls_lock.sh', 'cls/test_cls_log.sh', 'cls/test_cls_refcount.sh', 'cls/test_cls_rgw.sh', 'cls/test_cls_cmpomap.sh', 'cls/test_cls_2pc_queue.sh', 'rgw/test_rgw_gc_log.sh', 'rgw/test_rgw_obj.sh', 'rgw/test_rgw_throttle.sh', 'rgw/test_librgw_file.sh'] on client.0... 2023-09-22T13:15:33.536 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-22T13:15:33.834 DEBUG:teuthology.parallel:result is None 2023-09-22T13:15:33.834 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:15:33.861 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:15:33.861 DEBUG:teuthology.orchestra.run.smithi098:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-22T13:15:33.919 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-22T13:15:33.919 INFO:teuthology.run_tasks:Running task ragweed... 2023-09-22T13:15:33.931 DEBUG:tasks.ragweed:ragweed config is {'client.0': {'default-branch': 'ceph-pacific', 'rgw_server': 'client.0', 'stages': 'prepare,check'}} 2023-09-22T13:15:34.007 INFO:tasks.ragweed:Downloading ragweed... 2023-09-22T13:15:34.008 INFO:tasks.ragweed:Using branch 'wip-cfsnyder-bucket-check-pacific' for ragweed 2023-09-22T13:15:34.008 DEBUG:teuthology.orchestra.run.smithi098:> git clone -b wip-cfsnyder-bucket-check-pacific https://git.ceph.com/ragweed.git /home/ubuntu/cephtest/ragweed 2023-09-22T13:15:34.025 INFO:teuthology.orchestra.run.smithi098.stderr:Cloning into '/home/ubuntu/cephtest/ragweed'... 2023-09-22T13:15:34.116 INFO:teuthology.orchestra.run.smithi098.stderr:fatal: Remote branch wip-cfsnyder-bucket-check-pacific not found in upstream origin 2023-09-22T13:15:34.116 DEBUG:teuthology.orchestra.run:got remote process result: 128 2023-09-22T13:15:34.117 INFO:tasks.ragweed:Using branch 'ceph-pacific' for ragweed 2023-09-22T13:15:34.117 DEBUG:teuthology.orchestra.run.smithi098:> git clone -b ceph-pacific https://git.ceph.com/ragweed.git /home/ubuntu/cephtest/ragweed 2023-09-22T13:15:34.175 INFO:teuthology.orchestra.run.smithi098.stderr:Cloning into '/home/ubuntu/cephtest/ragweed'... 2023-09-22T13:15:34.318 INFO:tasks.ragweed:Creating rgw users... 2023-09-22T13:15:34.318 DEBUG:tasks.ragweed:Creating user ragweed.client.0 on client.0 2023-09-22T13:15:34.318 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user create --uid ragweed.client.0 --display-name 'Mr. ragweed.client.0' --access-key ESLCOESEKEOPHKQNAAJJ --secret PpNrGtp8hFcBuCGkRr7o2hW9oaljFL+YJv+Af8uCE+ptXHuwD3j2nA== --email ragweed.client.0+test@test.test --system false 2023-09-22T13:15:34.407 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:34.407 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:34.413 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.413+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.415 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.415+0000 7f10385dd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.415 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.415+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.416 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.416+0000 7f10385dd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.416 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.416+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.416 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.416+0000 7f10385dd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.416 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.416+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.416 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.417+0000 7f10385dd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:34.426 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.426+0000 7f10385dd500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:15:34.427 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.426+0000 7f10385dd500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:15:34.427 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.426+0000 7f10385dd500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:15:34.427 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.426+0000 7f10385dd500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:15:34.427 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.426+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.427 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.427+0000 7f10385dd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:34.427 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.427+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.427 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.427+0000 7f10385dd500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:15:34.427 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.427+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.429 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.427+0000 7f10385dd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:34.430 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.427+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.430 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.427+0000 7f10385dd500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:15:34.430 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.427+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.430 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.428+0000 7f10385dd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.430 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.428+0000 7f10385dd500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:15:34.430 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.428+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.430 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.428+0000 7f10385dd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.430 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.428+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.430 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.428+0000 7f10385dd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:34.430 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.428+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.428+0000 7f10385dd500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.428+0000 7f10385dd500 20 zonegroup default 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.428+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.429+0000 7f10385dd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.429+0000 7f10385dd500 10 Cannot find current period zone using local zone 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.429+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.429+0000 7f10385dd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.429+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.429+0000 7f10385dd500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.429+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.429+0000 7f10385dd500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:15:34.431 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.429+0000 7f10385dd500 20 zone default found 2023-09-22T13:15:34.432 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.429+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.432 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.430+0000 7f10385dd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.432 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.430+0000 7f10385dd500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.432 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.430+0000 7f10385dd500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.432 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.430+0000 7f10385dd500 20 started sync module instance, tier type = 2023-09-22T13:15:34.432 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.430+0000 7f10385dd500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:15:34.436 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.436+0000 7f10232be700 0 -- 172.21.15.98:0/2622874547 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7f101402c970 msgr2=0x7f10140183f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:15:34.436 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.436+0000 7f10385dd500 20 add_watcher() i=0 2023-09-22T13:15:34.436 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.436+0000 7f10385dd500 20 add_watcher() i=1 2023-09-22T13:15:34.436 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.436+0000 7f10385dd500 20 add_watcher() i=2 2023-09-22T13:15:34.436 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.436+0000 7f10385dd500 20 add_watcher() i=3 2023-09-22T13:15:34.436 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.436+0000 7f10385dd500 20 add_watcher() i=4 2023-09-22T13:15:34.437 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.436+0000 7f10385dd500 20 add_watcher() i=5 2023-09-22T13:15:34.437 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.436+0000 7f10385dd500 20 add_watcher() i=6 2023-09-22T13:15:34.437 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.437+0000 7f10385dd500 20 add_watcher() i=7 2023-09-22T13:15:34.437 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.437+0000 7f10385dd500 2 all 8 watchers are set, enabling cache 2023-09-22T13:15:34.438 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.439+0000 7f0ff57e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:15:34.439 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.439+0000 7f10385dd500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:15:34.439 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.439+0000 7f10385dd500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:15:34.441 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.441+0000 7f10232be700 0 -- 172.21.15.98:0/2936449324 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x558c530b1ea0 msgr2=0x558c530b7c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until injecting socket failure 2023-09-22T13:15:34.443 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.443+0000 7f10385dd500 5 note: GC not initialized 2023-09-22T13:15:34.445 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.444+0000 7f0f63fff700 20 reqs_thread_entry: start 2023-09-22T13:15:34.445 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.444+0000 7f0f61ffb700 20 reqs_thread_entry: start 2023-09-22T13:15:34.445 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.444+0000 7f10385dd500 20 init_complete bucket index max shards: 1 2023-09-22T13:15:34.445 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f10385dd500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:15:34.445 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f0f4ffff700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:16:04 2023 2023-09-22T13:15:34.445 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-22T13:15:34.445 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f0f4f7fe700 20 reqs_thread_entry: start 2023-09-22T13:15:34.445 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f10385dd500 20 get_system_obj_state: rctx=0x7fffb9a4d208 obj=default.rgw.meta:users.uid:ragweed.client.0 state=0x558c52f50810 s->prefetch_data=0 2023-09-22T13:15:34.445 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f10385dd500 10 cache get: name=default.rgw.meta+users.uid+ragweed.client.0 : miss 2023-09-22T13:15:34.446 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f10385dd500 10 cache put: name=default.rgw.meta+users.uid+ragweed.client.0 info.flags=0x0 2023-09-22T13:15:34.446 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f10385dd500 10 adding default.rgw.meta+users.uid+ragweed.client.0 to cache LRU end 2023-09-22T13:15:34.446 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f10385dd500 20 get_system_obj_state: rctx=0x7fffb9a4d218 obj=default.rgw.meta:users.email:ragweed.client.0+test@test.test state=0x558c52f50810 s->prefetch_data=0 2023-09-22T13:15:34.446 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f10385dd500 10 cache get: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test : miss 2023-09-22T13:15:34.447 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f10385dd500 10 cache put: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test info.flags=0x0 2023-09-22T13:15:34.447 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f10385dd500 10 adding default.rgw.meta+users.email+ragweed.client.0+test@test.test to cache LRU end 2023-09-22T13:15:34.447 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f10385dd500 20 get_system_obj_state: rctx=0x7fffb9a4d218 obj=default.rgw.meta:users.keys:ESLCOESEKEOPHKQNAAJJ state=0x558c52f50810 s->prefetch_data=0 2023-09-22T13:15:34.447 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.445+0000 7f10385dd500 10 cache get: name=default.rgw.meta+users.keys+ESLCOESEKEOPHKQNAAJJ : miss 2023-09-22T13:15:34.447 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.446+0000 7f10385dd500 10 cache put: name=default.rgw.meta+users.keys+ESLCOESEKEOPHKQNAAJJ info.flags=0x0 2023-09-22T13:15:34.447 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.446+0000 7f10385dd500 10 adding default.rgw.meta+users.keys+ESLCOESEKEOPHKQNAAJJ to cache LRU end 2023-09-22T13:15:34.447 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.446+0000 7f10385dd500 20 get_system_obj_state: rctx=0x7fffb9a4cb18 obj=default.rgw.meta:users.keys:ESLCOESEKEOPHKQNAAJJ state=0x558c52f50810 s->prefetch_data=0 2023-09-22T13:15:34.447 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.446+0000 7f10385dd500 10 cache get: name=default.rgw.meta+users.keys+ESLCOESEKEOPHKQNAAJJ : hit (negative entry) 2023-09-22T13:15:34.447 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.446+0000 7f10385dd500 20 get_system_obj_state: rctx=0x7fffb9a4ce58 obj=default.rgw.meta:users.keys:ESLCOESEKEOPHKQNAAJJ state=0x558c52f50810 s->prefetch_data=0 2023-09-22T13:15:34.447 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.446+0000 7f10385dd500 10 cache get: name=default.rgw.meta+users.keys+ESLCOESEKEOPHKQNAAJJ : hit (negative entry) 2023-09-22T13:15:34.448 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.447+0000 7f10385dd500 10 cache put: name=default.rgw.meta+users.uid+ragweed.client.0 info.flags=0x17 2023-09-22T13:15:34.448 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.447+0000 7f10385dd500 10 moving default.rgw.meta+users.uid+ragweed.client.0 to cache LRU end 2023-09-22T13:15:34.448 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.447+0000 7f10385dd500 10 distributing notification oid=default.rgw.control:notify.4 cni=[op: 0, obj: default.rgw.meta:users.uid:ragweed.client.0, ofs0, ns] 2023-09-22T13:15:34.448 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.447+0000 7f1021abb700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 996432412730 cookie 94061176936608 notifier 5460 bl.length()=641 2023-09-22T13:15:34.449 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.447+0000 7f1021abb700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+ragweed.client.0 info.flags=0x17 2023-09-22T13:15:34.449 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.447+0000 7f1021abb700 10 rgw watcher librados: moving default.rgw.meta+users.uid+ragweed.client.0 to cache LRU end 2023-09-22T13:15:34.449 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.448+0000 7f10385dd500 10 cache put: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test info.flags=0x7 2023-09-22T13:15:34.449 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.448+0000 7f10385dd500 10 moving default.rgw.meta+users.email+ragweed.client.0+test@test.test to cache LRU end 2023-09-22T13:15:34.449 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.448+0000 7f10385dd500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.email:ragweed.client.0+test@test.test, ofs0, ns] 2023-09-22T13:15:34.449 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.449+0000 7f10212ba700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 996432412677 cookie 94061176935312 notifier 5460 bl.length()=202 2023-09-22T13:15:34.449 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.449+0000 7f10212ba700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.email+ragweed.client.0+test@test.test info.flags=0x7 2023-09-22T13:15:34.450 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.449+0000 7f10212ba700 10 rgw watcher librados: moving default.rgw.meta+users.email+ragweed.client.0+test@test.test to cache LRU end 2023-09-22T13:15:34.450 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.450+0000 7f10385dd500 10 cache put: name=default.rgw.meta+users.keys+ESLCOESEKEOPHKQNAAJJ info.flags=0x7 2023-09-22T13:15:34.450 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.450+0000 7f10385dd500 10 moving default.rgw.meta+users.keys+ESLCOESEKEOPHKQNAAJJ to cache LRU end 2023-09-22T13:15:34.450 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.450+0000 7f10385dd500 10 distributing notification oid=default.rgw.control:notify.5 cni=[op: 0, obj: default.rgw.meta:users.keys:ESLCOESEKEOPHKQNAAJJ, ofs0, ns] 2023-09-22T13:15:34.451 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.450+0000 7f10212ba700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 996432412773 cookie 94061176940224 notifier 5460 bl.length()=190 2023-09-22T13:15:34.451 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.450+0000 7f10212ba700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+ESLCOESEKEOPHKQNAAJJ info.flags=0x7 2023-09-22T13:15:34.451 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.450+0000 7f10212ba700 10 rgw watcher librados: moving default.rgw.meta+users.keys+ESLCOESEKEOPHKQNAAJJ to cache LRU end 2023-09-22T13:15:34.451 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-22T13:15:34.451 INFO:teuthology.orchestra.run.smithi098.stdout: "user_id": "ragweed.client.0", 2023-09-22T13:15:34.451 INFO:teuthology.orchestra.run.smithi098.stdout: "display_name": "Mr. ragweed.client.0", 2023-09-22T13:15:34.451 INFO:teuthology.orchestra.run.smithi098.stdout: "email": "ragweed.client.0+test@test.test", 2023-09-22T13:15:34.451 INFO:teuthology.orchestra.run.smithi098.stdout: "suspended": 0, 2023-09-22T13:15:34.451 INFO:teuthology.orchestra.run.smithi098.stdout: "max_buckets": 1000, 2023-09-22T13:15:34.451 INFO:teuthology.orchestra.run.smithi098.stdout: "subusers": [], 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: "keys": [ 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: "user": "ragweed.client.0", 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: "access_key": "ESLCOESEKEOPHKQNAAJJ", 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: "secret_key": "PpNrGtp8hFcBuCGkRr7o2hW9oaljFL+YJv+Af8uCE+ptXHuwD3j2nA==" 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: ], 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: "swift_keys": [], 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: "caps": [], 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: "op_mask": "read, write, delete", 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: "default_placement": "", 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: "default_storage_class": "", 2023-09-22T13:15:34.452 INFO:teuthology.orchestra.run.smithi098.stdout: "placement_tags": [], 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: "bucket_quota": { 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: "enabled": false, 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: "check_on_raw": false, 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: "max_size": -1, 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: "max_size_kb": 0, 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: "max_objects": -1 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: "user_quota": { 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: "enabled": false, 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: "check_on_raw": false, 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: "max_size": -1, 2023-09-22T13:15:34.453 INFO:teuthology.orchestra.run.smithi098.stdout: "max_size_kb": 0, 2023-09-22T13:15:34.454 INFO:teuthology.orchestra.run.smithi098.stdout: "max_objects": -1 2023-09-22T13:15:34.454 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:15:34.454 INFO:teuthology.orchestra.run.smithi098.stdout: "temp_url_keys": [], 2023-09-22T13:15:34.454 INFO:teuthology.orchestra.run.smithi098.stdout: "type": "rgw", 2023-09-22T13:15:34.454 INFO:teuthology.orchestra.run.smithi098.stdout: "mfa_ids": [] 2023-09-22T13:15:34.454 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-22T13:15:34.454 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:15:34.455 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.452+0000 7f10385dd500 20 remove_watcher() i=0 2023-09-22T13:15:34.455 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.452+0000 7f10385dd500 2 removed watcher, disabling cache 2023-09-22T13:15:34.455 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.452+0000 7f10385dd500 20 remove_watcher() i=1 2023-09-22T13:15:34.455 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.453+0000 7f10385dd500 20 remove_watcher() i=2 2023-09-22T13:15:34.455 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.454+0000 7f10385dd500 20 remove_watcher() i=3 2023-09-22T13:15:34.455 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.455+0000 7f10385dd500 20 remove_watcher() i=4 2023-09-22T13:15:34.456 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.456+0000 7f10385dd500 20 remove_watcher() i=5 2023-09-22T13:15:34.456 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.456+0000 7f10385dd500 20 remove_watcher() i=6 2023-09-22T13:15:34.457 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.457+0000 7f10385dd500 20 remove_watcher() i=7 2023-09-22T13:15:34.457 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.458+0000 7f0ff57e2700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:15:34.462 DEBUG:tasks.ragweed:Creating user sysuser.client.0 on client.0 2023-09-22T13:15:34.462 DEBUG:teuthology.orchestra.run.smithi098:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin -n client.0 user create --uid sysuser.client.0 --display-name 'Mr. sysuser.client.0' --access-key BNKEEKGJLRBMWORSVNXK --secret y1q7JFLnK18RdfyDkpi/mQtAic7B/FrMJ0GjmVTRkuAfKRHRBoDfyg== --email sysuser.client.0+test@test.test --system true 2023-09-22T13:15:34.511 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setuser ceph since I am not root 2023-09-22T13:15:34.511 INFO:teuthology.orchestra.run.smithi098.stderr:ignoring --setgroup ceph since I am not root 2023-09-22T13:15:34.517 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.517+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.519 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.519+0000 7fccf2b2a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.519 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.519+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.520 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.520+0000 7fccf2b2a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.520 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.520+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.520 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.520+0000 7fccf2b2a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.520 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.520+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.520 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.521+0000 7fccf2b2a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:34.531 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.530+0000 7fccf2b2a500 20 RGWRados::pool_iterate: got zonegroup_info.3e5ed19c-ea6d-4aa0-8123-58909551c2c6 2023-09-22T13:15:34.531 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.530+0000 7fccf2b2a500 20 RGWRados::pool_iterate: got zone_names.default 2023-09-22T13:15:34.531 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.530+0000 7fccf2b2a500 20 RGWRados::pool_iterate: got zonegroups_names.default 2023-09-22T13:15:34.531 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.530+0000 7fccf2b2a500 20 RGWRados::pool_iterate: got zone_info.8b3505f7-cd1d-403d-9eb6-cc0558e0de01 2023-09-22T13:15:34.531 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.530+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.531 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.531+0000 7fccf2b2a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:34.531 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.531+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.531 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.531+0000 7fccf2b2a500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:15:34.531 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.531+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.531+0000 7fccf2b2a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.531+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.532+0000 7fccf2b2a500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.532+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.532+0000 7fccf2b2a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.532+0000 7fccf2b2a500 10 cannot find current period zonegroup using local zonegroup 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.532+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.532+0000 7fccf2b2a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.532+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.532+0000 7fccf2b2a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.532+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.533+0000 7fccf2b2a500 20 rados_obj.operate() r=0 bl.length=392 2023-09-22T13:15:34.533 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.533+0000 7fccf2b2a500 20 zonegroup default 2023-09-22T13:15:34.534 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.533+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.534 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.533+0000 7fccf2b2a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.534 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.533+0000 7fccf2b2a500 10 Cannot find current period zone using local zone 2023-09-22T13:15:34.534 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.533+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.534 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.533+0000 7fccf2b2a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.534 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.533+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.535 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.533+0000 7fccf2b2a500 20 rados_obj.operate() r=0 bl.length=46 2023-09-22T13:15:34.535 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.533+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.535 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.534+0000 7fccf2b2a500 20 rados_obj.operate() r=0 bl.length=1019 2023-09-22T13:15:34.535 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.534+0000 7fccf2b2a500 20 zone default found 2023-09-22T13:15:34.535 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.534+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.535 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.534+0000 7fccf2b2a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.535 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.534+0000 7fccf2b2a500 20 rados->read ofs=0 len=0 2023-09-22T13:15:34.536 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.534+0000 7fccf2b2a500 20 rados_obj.operate() r=-2 bl.length=0 2023-09-22T13:15:34.536 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.534+0000 7fccf2b2a500 20 started sync module instance, tier type = 2023-09-22T13:15:34.536 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.534+0000 7fccf2b2a500 20 started zone id=8b3505f7-cd1d-403d-9eb6-cc0558e0de01 (name=default) with tier type = 2023-09-22T13:15:34.540 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.541+0000 7fccf2b2a500 20 add_watcher() i=0 2023-09-22T13:15:34.541 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.541+0000 7fccf2b2a500 20 add_watcher() i=1 2023-09-22T13:15:34.541 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.541+0000 7fccf2b2a500 20 add_watcher() i=2 2023-09-22T13:15:34.541 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.541+0000 7fccf2b2a500 20 add_watcher() i=3 2023-09-22T13:15:34.541 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.541+0000 7fccf2b2a500 20 add_watcher() i=4 2023-09-22T13:15:34.541 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.541+0000 7fccf2b2a500 20 add_watcher() i=5 2023-09-22T13:15:34.541 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.541+0000 7fccf2b2a500 20 add_watcher() i=6 2023-09-22T13:15:34.541 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.541+0000 7fccdd80b700 0 -- 172.21.15.98:0/3292114576 >> [v2:172.21.15.169:6816/140717,v1:172.21.15.169:6817/140717] conn(0x7fccd40530c0 msgr2=0x7fccd4015750 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:15:34.542 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.542+0000 7fccf2b2a500 20 add_watcher() i=7 2023-09-22T13:15:34.542 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.542+0000 7fccf2b2a500 2 all 8 watchers are set, enabling cache 2023-09-22T13:15:34.543 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.543+0000 7fccaffdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:15:34.543 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.543+0000 7fccf2b2a500 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2023-09-22T13:15:34.543 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.543+0000 7fccf2b2a500 20 check_secure_mon_conn(): mode 1 is insecure 2023-09-22T13:15:34.546 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.546+0000 7fccdd80b700 0 -- 172.21.15.98:0/4260541864 >> [v2:172.21.15.98:3300/0,v1:172.21.15.98:6789/0] conn(0x55e17b2e6db0 msgr2=0x55e17b2f4680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1)._try_send injecting socket failure 2023-09-22T13:15:34.548 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.548+0000 7fccf2b2a500 5 note: GC not initialized 2023-09-22T13:15:34.549 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.549+0000 7fcc1effd700 20 reqs_thread_entry: start 2023-09-22T13:15:34.549 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.549+0000 7fcc1cff9700 20 reqs_thread_entry: start 2023-09-22T13:15:34.549 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.549+0000 7fccf2b2a500 20 init_complete bucket index max shards: 1 2023-09-22T13:15:34.549 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.549+0000 7fccf2b2a500 10 rgw notify: Started notification manager with: 1 workers 2023-09-22T13:15:34.549 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.549+0000 7fcc12ffd700 20 rgw notify: INFO: next queues processing will happen at: Fri Sep 22 13:16:04 2023 2023-09-22T13:15:34.550 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-22T13:15:34.550 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.549+0000 7fcc127fc700 20 reqs_thread_entry: start 2023-09-22T13:15:34.550 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.549+0000 7fccf2b2a500 20 get_system_obj_state: rctx=0x7ffcf0a084a8 obj=default.rgw.meta:users.uid:sysuser.client.0 state=0x55e17b185810 s->prefetch_data=0 2023-09-22T13:15:34.550 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.549+0000 7fccf2b2a500 10 cache get: name=default.rgw.meta+users.uid+sysuser.client.0 : miss 2023-09-22T13:15:34.551 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 10 cache put: name=default.rgw.meta+users.uid+sysuser.client.0 info.flags=0x0 2023-09-22T13:15:34.551 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 10 adding default.rgw.meta+users.uid+sysuser.client.0 to cache LRU end 2023-09-22T13:15:34.551 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 20 get_system_obj_state: rctx=0x7ffcf0a084b8 obj=default.rgw.meta:users.email:sysuser.client.0+test@test.test state=0x55e17b185810 s->prefetch_data=0 2023-09-22T13:15:34.551 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 10 cache get: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test : miss 2023-09-22T13:15:34.551 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 10 cache put: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test info.flags=0x0 2023-09-22T13:15:34.551 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 10 adding default.rgw.meta+users.email+sysuser.client.0+test@test.test to cache LRU end 2023-09-22T13:15:34.551 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 20 get_system_obj_state: rctx=0x7ffcf0a084b8 obj=default.rgw.meta:users.keys:BNKEEKGJLRBMWORSVNXK state=0x55e17b185810 s->prefetch_data=0 2023-09-22T13:15:34.551 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 10 cache get: name=default.rgw.meta+users.keys+BNKEEKGJLRBMWORSVNXK : miss 2023-09-22T13:15:34.551 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 10 cache put: name=default.rgw.meta+users.keys+BNKEEKGJLRBMWORSVNXK info.flags=0x0 2023-09-22T13:15:34.552 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 10 adding default.rgw.meta+users.keys+BNKEEKGJLRBMWORSVNXK to cache LRU end 2023-09-22T13:15:34.552 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 20 get_system_obj_state: rctx=0x7ffcf0a07db8 obj=default.rgw.meta:users.keys:BNKEEKGJLRBMWORSVNXK state=0x55e17b185810 s->prefetch_data=0 2023-09-22T13:15:34.552 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 10 cache get: name=default.rgw.meta+users.keys+BNKEEKGJLRBMWORSVNXK : hit (negative entry) 2023-09-22T13:15:34.552 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 20 get_system_obj_state: rctx=0x7ffcf0a080f8 obj=default.rgw.meta:users.keys:BNKEEKGJLRBMWORSVNXK state=0x55e17b185810 s->prefetch_data=0 2023-09-22T13:15:34.552 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.550+0000 7fccf2b2a500 10 cache get: name=default.rgw.meta+users.keys+BNKEEKGJLRBMWORSVNXK : hit (negative entry) 2023-09-22T13:15:34.552 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.551+0000 7fccf2b2a500 10 cache put: name=default.rgw.meta+users.uid+sysuser.client.0 info.flags=0x17 2023-09-22T13:15:34.553 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.551+0000 7fccf2b2a500 10 moving default.rgw.meta+users.uid+sysuser.client.0 to cache LRU end 2023-09-22T13:15:34.553 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.551+0000 7fccf2b2a500 10 distributing notification oid=default.rgw.control:notify.2 cni=[op: 0, obj: default.rgw.meta:users.uid:sysuser.client.0, ofs0, ns] 2023-09-22T13:15:34.553 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.552+0000 7fcccffff700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 996432412678 cookie 94426922563504 notifier 5475 bl.length()=641 2023-09-22T13:15:34.553 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.552+0000 7fcccffff700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.uid+sysuser.client.0 info.flags=0x17 2023-09-22T13:15:34.553 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.552+0000 7fcccffff700 10 rgw watcher librados: moving default.rgw.meta+users.uid+sysuser.client.0 to cache LRU end 2023-09-22T13:15:34.553 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.553+0000 7fccf2b2a500 10 cache put: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test info.flags=0x7 2023-09-22T13:15:34.553 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.553+0000 7fccf2b2a500 10 moving default.rgw.meta+users.email+sysuser.client.0+test@test.test to cache LRU end 2023-09-22T13:15:34.554 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.553+0000 7fccf2b2a500 10 distributing notification oid=default.rgw.control:notify.4 cni=[op: 0, obj: default.rgw.meta:users.email:sysuser.client.0+test@test.test, ofs0, ns] 2023-09-22T13:15:34.554 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.553+0000 7fcccffff700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 996432412731 cookie 94426922559904 notifier 5475 bl.length()=202 2023-09-22T13:15:34.554 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.553+0000 7fcccffff700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.email+sysuser.client.0+test@test.test info.flags=0x7 2023-09-22T13:15:34.554 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.553+0000 7fcccffff700 10 rgw watcher librados: moving default.rgw.meta+users.email+sysuser.client.0+test@test.test to cache LRU end 2023-09-22T13:15:34.554 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.555+0000 7fccf2b2a500 10 cache put: name=default.rgw.meta+users.keys+BNKEEKGJLRBMWORSVNXK info.flags=0x7 2023-09-22T13:15:34.554 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.555+0000 7fccf2b2a500 10 moving default.rgw.meta+users.keys+BNKEEKGJLRBMWORSVNXK to cache LRU end 2023-09-22T13:15:34.555 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.555+0000 7fccf2b2a500 10 distributing notification oid=default.rgw.control:notify.7 cni=[op: 0, obj: default.rgw.meta:users.keys:BNKEEKGJLRBMWORSVNXK, ofs0, ns] 2023-09-22T13:15:34.555 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.555+0000 7fcccf7fe700 10 rgw watcher librados: RGWWatcher::handle_notify() notify_id 996432412774 cookie 94426922579424 notifier 5475 bl.length()=190 2023-09-22T13:15:34.555 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.555+0000 7fcccf7fe700 10 rgw watcher librados: cache put: name=default.rgw.meta+users.keys+BNKEEKGJLRBMWORSVNXK info.flags=0x7 2023-09-22T13:15:34.555 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.555+0000 7fcccf7fe700 10 rgw watcher librados: moving default.rgw.meta+users.keys+BNKEEKGJLRBMWORSVNXK to cache LRU end 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout:{ 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout: "user_id": "sysuser.client.0", 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout: "display_name": "Mr. sysuser.client.0", 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout: "email": "sysuser.client.0+test@test.test", 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout: "suspended": 0, 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout: "max_buckets": 1000, 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout: "subusers": [], 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout: "keys": [ 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout: { 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout: "user": "sysuser.client.0", 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout: "access_key": "BNKEEKGJLRBMWORSVNXK", 2023-09-22T13:15:34.556 INFO:teuthology.orchestra.run.smithi098.stdout: "secret_key": "y1q7JFLnK18RdfyDkpi/mQtAic7B/FrMJ0GjmVTRkuAfKRHRBoDfyg==" 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: } 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: ], 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "swift_keys": [], 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "caps": [], 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "op_mask": "read, write, delete", 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "system": "true", 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "default_placement": "", 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "default_storage_class": "", 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "placement_tags": [], 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "bucket_quota": { 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "enabled": false, 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "check_on_raw": false, 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "max_size": -1, 2023-09-22T13:15:34.557 INFO:teuthology.orchestra.run.smithi098.stdout: "max_size_kb": 0, 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: "max_objects": -1 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: "user_quota": { 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: "enabled": false, 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: "check_on_raw": false, 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: "max_size": -1, 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: "max_size_kb": 0, 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: "max_objects": -1 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: }, 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: "temp_url_keys": [], 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: "type": "rgw", 2023-09-22T13:15:34.558 INFO:teuthology.orchestra.run.smithi098.stdout: "mfa_ids": [] 2023-09-22T13:15:34.559 INFO:teuthology.orchestra.run.smithi098.stdout:} 2023-09-22T13:15:34.559 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:15:34.559 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.557+0000 7fccf2b2a500 20 remove_watcher() i=0 2023-09-22T13:15:34.560 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.557+0000 7fccf2b2a500 2 removed watcher, disabling cache 2023-09-22T13:15:34.560 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.557+0000 7fccf2b2a500 20 remove_watcher() i=1 2023-09-22T13:15:34.560 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.558+0000 7fccf2b2a500 20 remove_watcher() i=2 2023-09-22T13:15:34.560 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.559+0000 7fccf2b2a500 20 remove_watcher() i=3 2023-09-22T13:15:34.560 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.560+0000 7fccf2b2a500 20 remove_watcher() i=4 2023-09-22T13:15:34.560 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.561+0000 7fccf2b2a500 20 remove_watcher() i=5 2023-09-22T13:15:34.561 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.561+0000 7fccf2b2a500 20 remove_watcher() i=6 2023-09-22T13:15:34.562 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.562+0000 7fccf2b2a500 20 remove_watcher() i=7 2023-09-22T13:15:34.562 INFO:teuthology.orchestra.run.smithi098.stderr:2023-09-22T13:15:34.563+0000 7fccaffdf700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2023-09-22T13:15:34.567 INFO:tasks.ragweed:Configuring ragweed... 2023-09-22T13:15:34.567 DEBUG:teuthology.orchestra.run.smithi098:> cd /home/ubuntu/cephtest/ragweed && ./bootstrap 2023-09-22T13:15:34.584 INFO:teuthology.orchestra.run.smithi098.stderr:+ set -e 2023-09-22T13:15:34.585 INFO:teuthology.orchestra.run.smithi098.stderr:+ '[' -f /etc/debian_version ']' 2023-09-22T13:15:34.585 INFO:teuthology.orchestra.run.smithi098.stderr:+ '[' -f /etc/redhat-release ']' 2023-09-22T13:15:34.585 INFO:teuthology.orchestra.run.smithi098.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:15:34.585 INFO:teuthology.orchestra.run.smithi098.stderr:++ rpm -qa python3-pip 2023-09-22T13:15:35.718 INFO:teuthology.orchestra.run.smithi098.stderr:+ '[' python3-pip-9.0.3-22.el8.noarch == '' ']' 2023-09-22T13:15:35.719 INFO:teuthology.orchestra.run.smithi098.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:15:35.719 INFO:teuthology.orchestra.run.smithi098.stderr:++ rpm -qa python3-virtualenv 2023-09-22T13:15:36.855 INFO:teuthology.orchestra.run.smithi098.stderr:+ '[' python3-virtualenv-15.1.0-21.module_el8.5.0+895+a459eca8.noarch == '' ']' 2023-09-22T13:15:36.855 INFO:teuthology.orchestra.run.smithi098.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:15:36.855 INFO:teuthology.orchestra.run.smithi098.stderr:++ rpm -qa python3-devel 2023-09-22T13:15:37.988 INFO:teuthology.orchestra.run.smithi098.stderr:+ '[' '' == '' ']' 2023-09-22T13:15:37.989 INFO:teuthology.orchestra.run.smithi098.stderr:+ missing=python3-devel 2023-09-22T13:15:37.989 INFO:teuthology.orchestra.run.smithi098.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:15:37.989 INFO:teuthology.orchestra.run.smithi098.stderr:++ rpm -qa libevent-devel 2023-09-22T13:15:39.126 INFO:teuthology.orchestra.run.smithi098.stderr:+ '[' '' == '' ']' 2023-09-22T13:15:39.126 INFO:teuthology.orchestra.run.smithi098.stderr:+ missing='python3-devel libevent-devel' 2023-09-22T13:15:39.126 INFO:teuthology.orchestra.run.smithi098.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:15:39.126 INFO:teuthology.orchestra.run.smithi098.stderr:++ rpm -qa libxml2-devel 2023-09-22T13:15:40.284 INFO:teuthology.orchestra.run.smithi098.stderr:+ '[' '' == '' ']' 2023-09-22T13:15:40.284 INFO:teuthology.orchestra.run.smithi098.stderr:+ missing='python3-devel libevent-devel libxml2-devel' 2023-09-22T13:15:40.284 INFO:teuthology.orchestra.run.smithi098.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:15:40.284 INFO:teuthology.orchestra.run.smithi098.stderr:++ rpm -qa libxslt-devel 2023-09-22T13:15:41.434 INFO:teuthology.orchestra.run.smithi098.stderr:+ '[' '' == '' ']' 2023-09-22T13:15:41.434 INFO:teuthology.orchestra.run.smithi098.stderr:+ missing='python3-devel libevent-devel libxml2-devel libxslt-devel' 2023-09-22T13:15:41.434 INFO:teuthology.orchestra.run.smithi098.stderr:+ for package in python3-pip python3-virtualenv python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:15:41.434 INFO:teuthology.orchestra.run.smithi098.stderr:++ rpm -qa zlib-devel 2023-09-22T13:15:42.572 INFO:teuthology.orchestra.run.smithi098.stderr:+ '[' '' == '' ']' 2023-09-22T13:15:42.572 INFO:teuthology.orchestra.run.smithi098.stderr:+ missing='python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel' 2023-09-22T13:15:42.572 INFO:teuthology.orchestra.run.smithi098.stderr:+ '[' -n 'python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel' ']' 2023-09-22T13:15:42.572 INFO:teuthology.orchestra.run.smithi098.stderr:+ echo './bootstrap: missing required RPM packages. Installing via sudo.' 2023-09-22T13:15:42.572 INFO:teuthology.orchestra.run.smithi098.stderr:./bootstrap: missing required RPM packages. Installing via sudo. 2023-09-22T13:15:42.572 INFO:teuthology.orchestra.run.smithi098.stderr:+ sudo yum -y install python3-devel libevent-devel libxml2-devel libxslt-devel zlib-devel 2023-09-22T13:15:43.420 INFO:teuthology.orchestra.run.smithi098.stdout:Last metadata expiration check: 0:29:36 ago on Fri 22 Sep 2023 12:46:07 PM UTC. 2023-09-22T13:15:44.820 INFO:teuthology.orchestra.run.smithi098.stdout:Package python36-devel-3.6.8-38.module_el8.5.0+895+a459eca8.x86_64 is already installed. 2023-09-22T13:15:45.010 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2023-09-22T13:15:45.011 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:15:45.012 INFO:teuthology.orchestra.run.smithi098.stdout: Package Arch Version Repository Size 2023-09-22T13:15:45.012 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:15:45.012 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2023-09-22T13:15:45.012 INFO:teuthology.orchestra.run.smithi098.stdout: libevent-devel x86_64 2.1.8-5.el8 appstream 104 k 2023-09-22T13:15:45.012 INFO:teuthology.orchestra.run.smithi098.stdout: libxml2-devel x86_64 2.9.7-16.el8 appstream 1.0 M 2023-09-22T13:15:45.012 INFO:teuthology.orchestra.run.smithi098.stdout: libxslt-devel x86_64 1.1.32-6.el8 appstream 323 k 2023-09-22T13:15:45.012 INFO:teuthology.orchestra.run.smithi098.stdout: zlib-devel x86_64 1.2.11-25.el8 baseos 59 k 2023-09-22T13:15:45.012 INFO:teuthology.orchestra.run.smithi098.stdout:Upgrading: 2023-09-22T13:15:45.012 INFO:teuthology.orchestra.run.smithi098.stdout: zlib x86_64 1.2.11-25.el8 baseos 103 k 2023-09-22T13:15:45.012 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2023-09-22T13:15:45.012 INFO:teuthology.orchestra.run.smithi098.stdout: cmake-filesystem x86_64 3.20.2-5.el8 appstream 45 k 2023-09-22T13:15:45.013 INFO:teuthology.orchestra.run.smithi098.stdout: libgcrypt-devel x86_64 1.8.5-7.el8 baseos 150 k 2023-09-22T13:15:45.013 INFO:teuthology.orchestra.run.smithi098.stdout: libgpg-error-devel x86_64 1.31-1.el8 baseos 64 k 2023-09-22T13:15:45.013 INFO:teuthology.orchestra.run.smithi098.stdout: xz-devel x86_64 5.2.4-4.el8 baseos 63 k 2023-09-22T13:15:45.013 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:15:45.013 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2023-09-22T13:15:45.013 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2023-09-22T13:15:45.013 INFO:teuthology.orchestra.run.smithi098.stdout:Install 8 Packages 2023-09-22T13:15:45.013 INFO:teuthology.orchestra.run.smithi098.stdout:Upgrade 1 Package 2023-09-22T13:15:45.013 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:15:45.014 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 1.9 M 2023-09-22T13:15:45.014 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2023-09-22T13:15:47.503 INFO:teuthology.orchestra.run.smithi098.stdout:(1/9): libevent-devel-2.1.8-5.el8.x86_64.rpm 46 kB/s | 104 kB 00:02 2023-09-22T13:15:49.244 INFO:teuthology.orchestra.run.smithi098.stdout:(2/9): cmake-filesystem-3.20.2-5.el8.x86_64.rpm 11 kB/s | 45 kB 00:03 2023-09-22T13:16:02.635 INFO:teuthology.orchestra.run.smithi098.stdout:(3/9): libxslt-devel-1.1.32-6.el8.x86_64.rpm 21 kB/s | 323 kB 00:15 2023-09-22T13:16:25.626 INFO:teuthology.orchestra.run.smithi098.stdout:(4/9): libgcrypt-devel-1.8.5-7.el8.x86_64.rpm 4.1 kB/s | 150 kB 00:36 2023-09-22T13:16:26.209 INFO:teuthology.orchestra.run.smithi098.stdout:(5/9): xz-devel-5.2.4-4.el8.x86_64.rpm 107 kB/s | 63 kB 00:00 2023-09-22T13:16:27.351 INFO:teuthology.orchestra.run.smithi098.stdout:(6/9): zlib-devel-1.2.11-25.el8.x86_64.rpm 51 kB/s | 59 kB 00:01 2023-09-22T13:16:29.393 INFO:teuthology.orchestra.run.smithi098.stdout:(7/9): zlib-1.2.11-25.el8.x86_64.rpm 51 kB/s | 103 kB 00:02 2023-09-22T13:16:36.559 INFO:teuthology.orchestra.run.smithi098.stdout:(8/9): libxml2-devel-2.9.7-16.el8.x86_64.rpm 21 kB/s | 1.0 MB 00:51 2023-09-22T13:16:37.476 INFO:teuthology.orchestra.run.smithi098.stdout:(9/9): libgpg-error-devel-1.31-1.el8.x86_64.rpm 1.8 kB/s | 64 kB 00:34 2023-09-22T13:16:37.476 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2023-09-22T13:16:37.476 INFO:teuthology.orchestra.run.smithi098.stdout:Total 38 kB/s | 1.9 MB 00:52 2023-09-22T13:16:37.573 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2023-09-22T13:16:37.703 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2023-09-22T13:16:37.703 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2023-09-22T13:16:37.819 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2023-09-22T13:16:37.822 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2023-09-22T13:16:38.202 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2023-09-22T13:16:38.342 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libgpg-error-devel-1.31-1.el8.x86_64 1/10 2023-09-22T13:16:38.376 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: libgpg-error-devel-1.31-1.el8.x86_64 1/10 2023-09-22T13:16:38.583 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libgcrypt-devel-1.8.5-7.el8.x86_64 2/10 2023-09-22T13:16:38.600 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: libgcrypt-devel-1.8.5-7.el8.x86_64 2/10 2023-09-22T13:16:38.814 INFO:teuthology.orchestra.run.smithi098.stdout: Upgrading : zlib-1.2.11-25.el8.x86_64 3/10 2023-09-22T13:16:38.980 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : zlib-devel-1.2.11-25.el8.x86_64 4/10 2023-09-22T13:16:39.158 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : xz-devel-5.2.4-4.el8.x86_64 5/10 2023-09-22T13:16:39.600 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : cmake-filesystem-3.20.2-5.el8.x86_64 6/10 2023-09-22T13:16:39.940 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libxml2-devel-2.9.7-16.el8.x86_64 7/10 2023-09-22T13:16:40.204 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libxslt-devel-1.1.32-6.el8.x86_64 8/10 2023-09-22T13:16:40.351 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : libevent-devel-2.1.8-5.el8.x86_64 9/10 2023-09-22T13:16:40.516 INFO:teuthology.orchestra.run.smithi098.stdout: Cleanup : zlib-1.2.11-21.el8.x86_64 10/10 2023-09-22T13:16:40.819 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: zlib-1.2.11-21.el8.x86_64 10/10 2023-09-22T13:16:40.819 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : cmake-filesystem-3.20.2-5.el8.x86_64 1/10 2023-09-22T13:16:40.819 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libevent-devel-2.1.8-5.el8.x86_64 2/10 2023-09-22T13:16:40.820 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libxml2-devel-2.9.7-16.el8.x86_64 3/10 2023-09-22T13:16:40.820 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libxslt-devel-1.1.32-6.el8.x86_64 4/10 2023-09-22T13:16:40.820 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libgcrypt-devel-1.8.5-7.el8.x86_64 5/10 2023-09-22T13:16:40.820 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : libgpg-error-devel-1.31-1.el8.x86_64 6/10 2023-09-22T13:16:40.820 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : xz-devel-5.2.4-4.el8.x86_64 7/10 2023-09-22T13:16:40.820 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : zlib-devel-1.2.11-25.el8.x86_64 8/10 2023-09-22T13:16:40.820 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : zlib-1.2.11-25.el8.x86_64 9/10 2023-09-22T13:16:41.506 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : zlib-1.2.11-21.el8.x86_64 10/10 2023-09-22T13:16:41.506 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:16:41.506 INFO:teuthology.orchestra.run.smithi098.stdout:Upgraded: 2023-09-22T13:16:41.506 INFO:teuthology.orchestra.run.smithi098.stdout: zlib-1.2.11-25.el8.x86_64 2023-09-22T13:16:41.506 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2023-09-22T13:16:41.506 INFO:teuthology.orchestra.run.smithi098.stdout: cmake-filesystem-3.20.2-5.el8.x86_64 libevent-devel-2.1.8-5.el8.x86_64 2023-09-22T13:16:41.506 INFO:teuthology.orchestra.run.smithi098.stdout: libgcrypt-devel-1.8.5-7.el8.x86_64 libgpg-error-devel-1.31-1.el8.x86_64 2023-09-22T13:16:41.506 INFO:teuthology.orchestra.run.smithi098.stdout: libxml2-devel-2.9.7-16.el8.x86_64 libxslt-devel-1.1.32-6.el8.x86_64 2023-09-22T13:16:41.507 INFO:teuthology.orchestra.run.smithi098.stdout: xz-devel-5.2.4-4.el8.x86_64 zlib-devel-1.2.11-25.el8.x86_64 2023-09-22T13:16:41.507 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:16:41.507 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2023-09-22T13:16:41.697 INFO:teuthology.orchestra.run.smithi098.stderr:+ sudo pip3 install --upgrade --trusted-host apt-mirror.front.sepia.ceph.com setuptools cffi cachecontrol 2023-09-22T13:16:42.017 INFO:teuthology.orchestra.run.smithi098.stderr:WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead. 2023-09-22T13:16:42.643 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting setuptools 2023-09-22T13:16:43.022 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading https://files.pythonhosted.org/packages/b0/3a/88b210db68e56854d0bcf4b38e165e03be377e13907746f825790f3df5bf/setuptools-59.6.0-py3-none-any.whl (952kB) 2023-09-22T13:16:43.524 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting cffi 2023-09-22T13:16:43.749 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading https://files.pythonhosted.org/packages/3a/12/d6066828014b9ccb2bbb8e1d9dc28872d20669b65aeb4a86806a0757813f/cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (402kB) 2023-09-22T13:16:43.851 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting cachecontrol 2023-09-22T13:16:43.965 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading https://files.pythonhosted.org/packages/72/a2/28e0ef082f7d78253aded97933e1d7b94bab3c5be366e8afd6513de4028e/CacheControl-0.12.14-py2.py3-none-any.whl 2023-09-22T13:16:44.116 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting pycparser (from cffi) 2023-09-22T13:16:44.141 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading https://files.pythonhosted.org/packages/62/d5/5f610ebe421e85889f2e55e33b7f9a6795bd982198517d912eb1c76e1a53/pycparser-2.21-py2.py3-none-any.whl (118kB) 2023-09-22T13:16:44.335 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting requests (from cachecontrol) 2023-09-22T13:16:44.408 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading https://files.pythonhosted.org/packages/2d/61/08076519c80041bc0ffa1a8af0cbd3bf3e2b62af10435d269a9d0f40564d/requests-2.27.1-py2.py3-none-any.whl (63kB) 2023-09-22T13:16:44.506 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting msgpack>=0.5.2 (from cachecontrol) 2023-09-22T13:16:44.707 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading https://files.pythonhosted.org/packages/dc/a1/eba11a0d4b764bc62966a565b470f8c6f38242723ba3057e9b5098678c30/msgpack-1.0.5.tar.gz (127kB) 2023-09-22T13:16:45.226 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting urllib3<1.27,>=1.21.1 (from requests->cachecontrol) 2023-09-22T13:16:45.295 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading https://files.pythonhosted.org/packages/c5/05/c214b32d21c0b465506f95c4f28ccbcba15022e000b043b72b3df7728471/urllib3-1.26.16-py2.py3-none-any.whl (143kB) 2023-09-22T13:16:45.425 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting certifi>=2017.4.17 (from requests->cachecontrol) 2023-09-22T13:16:45.466 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl (158kB) 2023-09-22T13:16:45.541 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting charset-normalizer~=2.0.0; python_version >= "3" (from requests->cachecontrol) 2023-09-22T13:16:45.752 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading https://files.pythonhosted.org/packages/06/b3/24afc8868eba069a7f03650ac750a778862dc34941a4bebeb58706715726/charset_normalizer-2.0.12-py3-none-any.whl 2023-09-22T13:16:45.929 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting idna<4,>=2.5; python_version >= "3" (from requests->cachecontrol) 2023-09-22T13:16:45.959 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl (61kB) 2023-09-22T13:16:45.968 INFO:teuthology.orchestra.run.smithi098.stdout:Installing collected packages: setuptools, pycparser, cffi, urllib3, certifi, charset-normalizer, idna, requests, msgpack, cachecontrol 2023-09-22T13:16:46.481 INFO:teuthology.orchestra.run.smithi098.stdout: Running setup.py install for msgpack: started 2023-09-22T13:16:46.762 INFO:teuthology.orchestra.run.smithi098.stdout: Running setup.py install for msgpack: finished with status 'done' 2023-09-22T13:16:46.805 INFO:teuthology.orchestra.run.smithi098.stdout:Successfully installed cachecontrol-0.12.14 certifi-2023.7.22 cffi-1.15.1 charset-normalizer-2.0.12 idna-3.4 msgpack-1.0.5 pycparser-2.21 requests-2.27.1 setuptools-59.6.0 urllib3-1.26.16 2023-09-22T13:16:46.928 INFO:teuthology.orchestra.run.smithi098.stderr:+ virtualenv -p python3 --system-site-packages --download virtualenv 2023-09-22T13:16:47.331 INFO:teuthology.orchestra.run.smithi098.stdout:Using base prefix '/usr' 2023-09-22T13:16:47.331 INFO:teuthology.orchestra.run.smithi098.stdout:New python executable in /home/ubuntu/cephtest/ragweed/virtualenv/bin/python3 2023-09-22T13:16:47.331 INFO:teuthology.orchestra.run.smithi098.stdout:Also creating executable in /home/ubuntu/cephtest/ragweed/virtualenv/bin/python 2023-09-22T13:16:50.259 INFO:teuthology.orchestra.run.smithi098.stdout:Installing setuptools, pip, wheel...done. 2023-09-22T13:16:50.268 INFO:teuthology.orchestra.run.smithi098.stdout:Running virtualenv with interpreter /usr/bin/python3 2023-09-22T13:16:50.282 INFO:teuthology.orchestra.run.smithi098.stderr:+ ./virtualenv/bin/pip install --upgrade pip 2023-09-22T13:16:50.653 INFO:teuthology.orchestra.run.smithi098.stdout:Requirement already satisfied: pip in ./virtualenv/lib/python3.6/site-packages (21.3.1) 2023-09-22T13:16:51.281 INFO:teuthology.orchestra.run.smithi098.stderr:+ ./virtualenv/bin/pip install six 2023-09-22T13:16:51.646 INFO:teuthology.orchestra.run.smithi098.stdout:Requirement already satisfied: six in /usr/lib/python3.6/site-packages (1.11.0) 2023-09-22T13:16:51.823 INFO:teuthology.orchestra.run.smithi098.stderr:+ ./virtualenv/bin/pip install -I nose 2023-09-22T13:16:52.310 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting nose 2023-09-22T13:16:52.372 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading nose-1.3.7-py3-none-any.whl (154 kB) 2023-09-22T13:16:52.561 INFO:teuthology.orchestra.run.smithi098.stdout:Installing collected packages: nose 2023-09-22T13:16:52.673 INFO:teuthology.orchestra.run.smithi098.stdout:Successfully installed nose-1.3.7 2023-09-22T13:16:52.725 INFO:teuthology.orchestra.run.smithi098.stderr:+ ./virtualenv/bin/pip install setuptools 2023-09-22T13:16:53.088 INFO:teuthology.orchestra.run.smithi098.stdout:Requirement already satisfied: setuptools in ./virtualenv/lib/python3.6/site-packages (59.6.0) 2023-09-22T13:16:53.273 INFO:teuthology.orchestra.run.smithi098.stderr:+ ./virtualenv/bin/pip install -U -r requirements.txt 2023-09-22T13:16:53.654 INFO:teuthology.orchestra.run.smithi098.stdout:Requirement already satisfied: PyYAML in /usr/lib64/python3.6/site-packages (from -r requirements.txt (line 1)) (3.12) 2023-09-22T13:16:53.860 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting PyYAML 2023-09-22T13:16:53.917 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (677 kB) 2023-09-22T13:16:54.018 INFO:teuthology.orchestra.run.smithi098.stdout:Requirement already satisfied: nose>=1.0.0 in ./virtualenv/lib/python3.6/site-packages (from -r requirements.txt (line 2)) (1.3.7) 2023-09-22T13:16:54.145 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting boto!=2.46.0,>=2.6.0 2023-09-22T13:16:54.166 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading boto-2.49.0-py2.py3-none-any.whl (1.4 MB) 2023-09-22T13:16:54.288 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting munch>=1.0.0 2023-09-22T13:16:54.306 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading munch-4.0.0-py2.py3-none-any.whl (9.9 kB) 2023-09-22T13:16:54.745 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting gevent>=1.0 2023-09-22T13:16:54.765 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading gevent-22.10.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.8 MB) 2023-09-22T13:16:54.969 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting httplib2 2023-09-22T13:16:54.987 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading httplib2-0.22.0-py3-none-any.whl (96 kB) 2023-09-22T13:16:55.045 INFO:teuthology.orchestra.run.smithi098.stdout:Requirement already satisfied: lxml in /usr/lib64/python3.6/site-packages (from -r requirements.txt (line 7)) (4.2.3) 2023-09-22T13:16:55.842 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting lxml 2023-09-22T13:16:55.866 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading lxml-4.9.3-cp36-cp36m-manylinux_2_28_x86_64.whl (7.4 MB) 2023-09-22T13:16:56.207 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting importlib-metadata>=1.7.0 2023-09-22T13:16:56.230 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading importlib_metadata-4.8.3-py3-none-any.whl (17 kB) 2023-09-22T13:16:56.659 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting zope.interface 2023-09-22T13:16:56.678 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading zope.interface-5.5.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (253 kB) 2023-09-22T13:16:56.761 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting zope.event 2023-09-22T13:16:56.886 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading zope.event-4.6-py2.py3-none-any.whl (6.8 kB) 2023-09-22T13:16:57.337 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting greenlet>=2.0.0 2023-09-22T13:16:57.364 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading greenlet-2.0.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (564 kB) 2023-09-22T13:16:57.410 INFO:teuthology.orchestra.run.smithi098.stdout:Requirement already satisfied: setuptools in ./virtualenv/lib/python3.6/site-packages (from gevent>=1.0->-r requirements.txt (line 5)) (59.6.0) 2023-09-22T13:16:57.528 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 2023-09-22T13:16:57.555 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading pyparsing-3.1.1-py3-none-any.whl (103 kB) 2023-09-22T13:16:57.730 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting zipp>=0.5 2023-09-22T13:16:57.787 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading zipp-3.6.0-py3-none-any.whl (5.3 kB) 2023-09-22T13:16:57.911 INFO:teuthology.orchestra.run.smithi098.stdout:Collecting typing-extensions>=3.6.4 2023-09-22T13:16:57.969 INFO:teuthology.orchestra.run.smithi098.stdout: Downloading typing_extensions-4.1.1-py3-none-any.whl (26 kB) 2023-09-22T13:16:58.263 INFO:teuthology.orchestra.run.smithi098.stdout:Installing collected packages: zipp, typing-extensions, zope.interface, zope.event, pyparsing, importlib-metadata, greenlet, PyYAML, munch, lxml, httplib2, gevent, boto 2023-09-22T13:16:58.579 INFO:teuthology.orchestra.run.smithi098.stdout: Attempting uninstall: pyparsing 2023-09-22T13:16:58.580 INFO:teuthology.orchestra.run.smithi098.stdout: Found existing installation: pyparsing 2.1.10 2023-09-22T13:16:58.580 INFO:teuthology.orchestra.run.smithi098.stdout: Not uninstalling pyparsing at /usr/lib/python3.6/site-packages, outside environment /home/ubuntu/cephtest/ragweed/virtualenv 2023-09-22T13:16:58.580 INFO:teuthology.orchestra.run.smithi098.stdout: Can't uninstall 'pyparsing'. No files were found to uninstall. 2023-09-22T13:16:58.782 INFO:teuthology.orchestra.run.smithi098.stdout: Attempting uninstall: PyYAML 2023-09-22T13:16:58.782 INFO:teuthology.orchestra.run.smithi098.stdout: Found existing installation: PyYAML 3.12 2023-09-22T13:16:58.782 INFO:teuthology.orchestra.run.smithi098.stdout: Not uninstalling pyyaml at /usr/lib64/python3.6/site-packages, outside environment /home/ubuntu/cephtest/ragweed/virtualenv 2023-09-22T13:16:58.783 INFO:teuthology.orchestra.run.smithi098.stdout: Can't uninstall 'PyYAML'. No files were found to uninstall. 2023-09-22T13:16:58.891 INFO:teuthology.orchestra.run.smithi098.stdout: Attempting uninstall: lxml 2023-09-22T13:16:58.892 INFO:teuthology.orchestra.run.smithi098.stdout: Found existing installation: lxml 4.2.3 2023-09-22T13:16:58.892 INFO:teuthology.orchestra.run.smithi098.stdout: Not uninstalling lxml at /usr/lib64/python3.6/site-packages, outside environment /home/ubuntu/cephtest/ragweed/virtualenv 2023-09-22T13:16:58.892 INFO:teuthology.orchestra.run.smithi098.stdout: Can't uninstall 'lxml'. No files were found to uninstall. 2023-09-22T13:17:00.046 INFO:teuthology.orchestra.run.smithi098.stdout:Successfully installed PyYAML-6.0.1 boto-2.49.0 gevent-22.10.2 greenlet-2.0.2 httplib2-0.22.0 importlib-metadata-4.8.3 lxml-4.9.3 munch-4.0.0 pyparsing-3.1.1 typing-extensions-4.1.1 zipp-3.6.0 zope.event-4.6 zope.interface-5.5.2 2023-09-22T13:17:00.145 INFO:teuthology.orchestra.run.smithi098.stderr:+ ./virtualenv/bin/python setup.py develop 2023-09-22T13:17:00.389 INFO:teuthology.orchestra.run.smithi098.stdout:running develop 2023-09-22T13:17:00.427 INFO:teuthology.orchestra.run.smithi098.stdout:running egg_info 2023-09-22T13:17:00.427 INFO:teuthology.orchestra.run.smithi098.stdout:creating ragweed.egg-info 2023-09-22T13:17:00.427 INFO:teuthology.orchestra.run.smithi098.stdout:writing ragweed.egg-info/PKG-INFO 2023-09-22T13:17:00.427 INFO:teuthology.orchestra.run.smithi098.stdout:writing dependency_links to ragweed.egg-info/dependency_links.txt 2023-09-22T13:17:00.427 INFO:teuthology.orchestra.run.smithi098.stdout:writing requirements to ragweed.egg-info/requires.txt 2023-09-22T13:17:00.427 INFO:teuthology.orchestra.run.smithi098.stdout:writing top-level names to ragweed.egg-info/top_level.txt 2023-09-22T13:17:00.427 INFO:teuthology.orchestra.run.smithi098.stdout:writing manifest file 'ragweed.egg-info/SOURCES.txt' 2023-09-22T13:17:00.428 INFO:teuthology.orchestra.run.smithi098.stdout:reading manifest file 'ragweed.egg-info/SOURCES.txt' 2023-09-22T13:17:00.428 INFO:teuthology.orchestra.run.smithi098.stdout:adding license file 'LICENSE' 2023-09-22T13:17:00.428 INFO:teuthology.orchestra.run.smithi098.stdout:writing manifest file 'ragweed.egg-info/SOURCES.txt' 2023-09-22T13:17:00.430 INFO:teuthology.orchestra.run.smithi098.stdout:running build_ext 2023-09-22T13:17:00.430 INFO:teuthology.orchestra.run.smithi098.stdout:Creating /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/ragweed.egg-link (link to .) 2023-09-22T13:17:00.430 INFO:teuthology.orchestra.run.smithi098.stdout:Adding ragweed 0.0.1 to easy-install.pth file 2023-09-22T13:17:00.431 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.431 INFO:teuthology.orchestra.run.smithi098.stdout:Installed /home/ubuntu/cephtest/ragweed 2023-09-22T13:17:00.432 INFO:teuthology.orchestra.run.smithi098.stdout:Processing dependencies for ragweed==0.0.1 2023-09-22T13:17:00.434 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for isodate>=0.4.4 2023-09-22T13:17:00.434 INFO:teuthology.orchestra.run.smithi098.stdout:Reading https://pypi.org/simple/isodate/ 2023-09-22T13:17:00.509 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading https://files.pythonhosted.org/packages/b6/85/7882d311924cbcfc70b1890780763e36ff0b140c7e51c110fc59a532f087/isodate-0.6.1-py2.py3-none-any.whl#sha256=0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96 2023-09-22T13:17:00.583 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: isodate 0.6.1 2023-09-22T13:17:00.583 INFO:teuthology.orchestra.run.smithi098.stdout:Processing isodate-0.6.1-py2.py3-none-any.whl 2023-09-22T13:17:00.589 INFO:teuthology.orchestra.run.smithi098.stdout:Installing isodate-0.6.1-py2.py3-none-any.whl to /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:17:00.595 INFO:teuthology.orchestra.run.smithi098.stdout:Adding isodate 0.6.1 to easy-install.pth file 2023-09-22T13:17:00.596 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.596 INFO:teuthology.orchestra.run.smithi098.stdout:Installed /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/isodate-0.6.1-py3.6.egg 2023-09-22T13:17:00.700 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for gevent==22.10.2 2023-09-22T13:17:00.700 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: gevent 22.10.2 2023-09-22T13:17:00.700 INFO:teuthology.orchestra.run.smithi098.stdout:Adding gevent 22.10.2 to easy-install.pth file 2023-09-22T13:17:00.700 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.700 INFO:teuthology.orchestra.run.smithi098.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:17:00.701 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for munch==4.0.0 2023-09-22T13:17:00.701 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: munch 4.0.0 2023-09-22T13:17:00.702 INFO:teuthology.orchestra.run.smithi098.stdout:Adding munch 4.0.0 to easy-install.pth file 2023-09-22T13:17:00.702 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.702 INFO:teuthology.orchestra.run.smithi098.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:17:00.702 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for PyYAML==6.0.1 2023-09-22T13:17:00.702 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: PyYAML 6.0.1 2023-09-22T13:17:00.702 INFO:teuthology.orchestra.run.smithi098.stdout:Adding PyYAML 6.0.1 to easy-install.pth file 2023-09-22T13:17:00.702 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.702 INFO:teuthology.orchestra.run.smithi098.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:17:00.703 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for boto==2.49.0 2023-09-22T13:17:00.703 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: boto 2.49.0 2023-09-22T13:17:00.703 INFO:teuthology.orchestra.run.smithi098.stdout:Adding boto 2.49.0 to easy-install.pth file 2023-09-22T13:17:00.703 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.704 INFO:teuthology.orchestra.run.smithi098.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:17:00.704 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for six==1.11.0 2023-09-22T13:17:00.704 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: six 1.11.0 2023-09-22T13:17:00.704 INFO:teuthology.orchestra.run.smithi098.stdout:Adding six 1.11.0 to easy-install.pth file 2023-09-22T13:17:00.704 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.704 INFO:teuthology.orchestra.run.smithi098.stdout:Using /usr/lib/python3.6/site-packages 2023-09-22T13:17:00.705 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for zope.event==4.6 2023-09-22T13:17:00.705 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: zope.event 4.6 2023-09-22T13:17:00.705 INFO:teuthology.orchestra.run.smithi098.stdout:Adding zope.event 4.6 to easy-install.pth file 2023-09-22T13:17:00.705 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.705 INFO:teuthology.orchestra.run.smithi098.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:17:00.706 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for setuptools==59.6.0 2023-09-22T13:17:00.706 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: setuptools 59.6.0 2023-09-22T13:17:00.706 INFO:teuthology.orchestra.run.smithi098.stdout:Adding setuptools 59.6.0 to easy-install.pth file 2023-09-22T13:17:00.706 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.706 INFO:teuthology.orchestra.run.smithi098.stdout:Using /usr/local/lib/python3.6/site-packages 2023-09-22T13:17:00.708 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for greenlet==2.0.2 2023-09-22T13:17:00.708 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: greenlet 2.0.2 2023-09-22T13:17:00.708 INFO:teuthology.orchestra.run.smithi098.stdout:Adding greenlet 2.0.2 to easy-install.pth file 2023-09-22T13:17:00.708 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.708 INFO:teuthology.orchestra.run.smithi098.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:17:00.708 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for zope.interface==5.5.2 2023-09-22T13:17:00.708 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: zope.interface 5.5.2 2023-09-22T13:17:00.708 INFO:teuthology.orchestra.run.smithi098.stdout:Adding zope.interface 5.5.2 to easy-install.pth file 2023-09-22T13:17:00.708 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.708 INFO:teuthology.orchestra.run.smithi098.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:17:00.710 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for importlib-metadata==4.8.3 2023-09-22T13:17:00.710 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: importlib-metadata 4.8.3 2023-09-22T13:17:00.710 INFO:teuthology.orchestra.run.smithi098.stdout:Adding importlib-metadata 4.8.3 to easy-install.pth file 2023-09-22T13:17:00.710 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.710 INFO:teuthology.orchestra.run.smithi098.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:17:00.711 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for typing-extensions==4.1.1 2023-09-22T13:17:00.711 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: typing-extensions 4.1.1 2023-09-22T13:17:00.711 INFO:teuthology.orchestra.run.smithi098.stdout:Adding typing-extensions 4.1.1 to easy-install.pth file 2023-09-22T13:17:00.711 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.711 INFO:teuthology.orchestra.run.smithi098.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:17:00.711 INFO:teuthology.orchestra.run.smithi098.stdout:Searching for zipp==3.6.0 2023-09-22T13:17:00.711 INFO:teuthology.orchestra.run.smithi098.stdout:Best match: zipp 3.6.0 2023-09-22T13:17:00.711 INFO:teuthology.orchestra.run.smithi098.stdout:Adding zipp 3.6.0 to easy-install.pth file 2023-09-22T13:17:00.711 INFO:teuthology.orchestra.run.smithi098.stdout: 2023-09-22T13:17:00.711 INFO:teuthology.orchestra.run.smithi098.stdout:Using /home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages 2023-09-22T13:17:00.711 INFO:teuthology.orchestra.run.smithi098.stdout:Finished processing dependencies for ragweed==0.0.1 2023-09-22T13:17:00.712 INFO:teuthology.orchestra.run.smithi098.stderr:/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/setuptools/command/easy_install.py:159: EasyInstallDeprecationWarning: easy_install command is deprecated. Use build and pip and other standards-based tools. 2023-09-22T13:17:00.712 INFO:teuthology.orchestra.run.smithi098.stderr: EasyInstallDeprecationWarning, 2023-09-22T13:17:00.712 INFO:teuthology.orchestra.run.smithi098.stderr:/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/setuptools/command/install.py:37: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools. 2023-09-22T13:17:00.712 INFO:teuthology.orchestra.run.smithi098.stderr: setuptools.SetuptoolsDeprecationWarning, 2023-09-22T13:17:00.712 INFO:teuthology.orchestra.run.smithi098.stderr:/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/pkg_resources/__init__.py:119: PkgResourcesDeprecationWarning: is an invalid version and will not be supported in a future release 2023-09-22T13:17:00.712 INFO:teuthology.orchestra.run.smithi098.stderr: PkgResourcesDeprecationWarning, 2023-09-22T13:17:00.731 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:17:00.732 DEBUG:teuthology.orchestra.run.smithi098:> dd of=/home/ubuntu/cephtest/archive/ragweed.client.0.conf 2023-09-22T13:17:00.789 INFO:tasks.ragweed:Configuring boto... 2023-09-22T13:17:00.790 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2023-09-22T13:17:00.790 DEBUG:teuthology.orchestra.run.smithi098:> dd of=/home/ubuntu/cephtest/boto.cfg 2023-09-22T13:17:00.848 DEBUG:teuthology.orchestra.run.smithi098:ragweed tests against rgw> RAGWEED_CONF=/home/ubuntu/cephtest/archive/ragweed.client.0.conf RAGWEED_STAGES=prepare,check BOTO_CONFIG=/home/ubuntu/cephtest/boto.cfg /home/ubuntu/cephtest/ragweed/virtualenv/bin/python -m nose -w /home/ubuntu/cephtest/ragweed -v -a '!fails_on_rgw' 2023-09-22T13:17:03.837 INFO:teuthology.orchestra.run.smithi098.stderr:ragweed.tests.tests.r_test_multipart_defer_complete.test ... ok 2023-09-22T13:17:04.405 INFO:teuthology.orchestra.run.smithi098.stderr:ragweed.tests.tests.r_test_multipart_defer_update_complete.test ... ok 2023-09-22T13:17:04.811 INFO:teuthology.orchestra.run.smithi098.stderr:ragweed.tests.tests.r_test_multipart_index_versioning.test ... ERROR 2023-09-22T13:17:05.278 INFO:teuthology.orchestra.run.smithi098.stderr:ragweed.tests.tests.r_test_multipart_simple.test ... ok 2023-09-22T13:17:09.674 INFO:teuthology.orchestra.run.smithi098.stderr:ragweed.tests.tests.r_test_obj_storage_class.test ... ok 2023-09-22T13:17:10.773 INFO:teuthology.orchestra.run.smithi098.stderr:ragweed.tests.tests.r_test_obj_storage_class_copy.test ... ok 2023-09-22T13:17:11.986 INFO:teuthology.orchestra.run.smithi098.stderr:ragweed.tests.tests.r_test_obj_storage_class_multipart.test ... ok 2023-09-22T13:17:14.100 INFO:teuthology.orchestra.run.smithi098.stderr:ragweed.tests.tests.r_test_obj_storage_class_multipart_copy.test ... ok 2023-09-22T13:17:14.535 INFO:teuthology.orchestra.run.smithi098.stderr:ragweed.tests.tests.r_test_small_obj_data.test ... ok 2023-09-22T13:17:14.535 INFO:teuthology.orchestra.run.smithi098.stderr: 2023-09-22T13:17:14.535 INFO:teuthology.orchestra.run.smithi098.stderr:====================================================================== 2023-09-22T13:17:14.535 INFO:teuthology.orchestra.run.smithi098.stderr:ERROR: ragweed.tests.tests.r_test_multipart_index_versioning.test 2023-09-22T13:17:14.535 INFO:teuthology.orchestra.run.smithi098.stderr:---------------------------------------------------------------------- 2023-09-22T13:17:14.535 INFO:teuthology.orchestra.run.smithi098.stderr:Traceback (most recent call last): 2023-09-22T13:17:14.536 INFO:teuthology.orchestra.run.smithi098.stderr: File "/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest 2023-09-22T13:17:14.536 INFO:teuthology.orchestra.run.smithi098.stderr: self.test(*self.arg) 2023-09-22T13:17:14.536 INFO:teuthology.orchestra.run.smithi098.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/framework.py", line 345, in test 2023-09-22T13:17:14.536 INFO:teuthology.orchestra.run.smithi098.stderr: self.check() 2023-09-22T13:17:14.536 INFO:teuthology.orchestra.run.smithi098.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/tests/tests.py", line 296, in check 2023-09-22T13:17:14.536 INFO:teuthology.orchestra.run.smithi098.stderr: index_check_result = rgwa().check_bucket_index(rb.name) 2023-09-22T13:17:14.536 INFO:teuthology.orchestra.run.smithi098.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/framework.py", line 68, in check_bucket_index 2023-09-22T13:17:14.536 INFO:teuthology.orchestra.run.smithi098.stderr: return self.get_resource('/admin/bucket',{'index' : None, 'bucket':bucket_name}) 2023-09-22T13:17:14.536 INFO:teuthology.orchestra.run.smithi098.stderr: File "/home/ubuntu/cephtest/ragweed/ragweed/framework.py", line 50, in get_resource 2023-09-22T13:17:14.536 INFO:teuthology.orchestra.run.smithi098.stderr: return munch.munchify(json.loads(r.read())) 2023-09-22T13:17:14.536 INFO:teuthology.orchestra.run.smithi098.stderr: File "/usr/lib64/python3.6/json/__init__.py", line 354, in loads 2023-09-22T13:17:14.537 INFO:teuthology.orchestra.run.smithi098.stderr: return _default_decoder.decode(s) 2023-09-22T13:17:14.537 INFO:teuthology.orchestra.run.smithi098.stderr: File "/usr/lib64/python3.6/json/decoder.py", line 342, in decode 2023-09-22T13:17:14.537 INFO:teuthology.orchestra.run.smithi098.stderr: raise JSONDecodeError("Extra data", s, end) 2023-09-22T13:17:14.537 INFO:teuthology.orchestra.run.smithi098.stderr:json.decoder.JSONDecodeError: Extra data: line 1 column 3 (char 2) 2023-09-22T13:17:14.537 INFO:teuthology.orchestra.run.smithi098.stderr:-------------------- >> begin captured stdout << --------------------- 2023-09-22T13:17:14.537 INFO:teuthology.orchestra.run.smithi098.stderr:Munch({'key': 'bucket:test-client.0-r-test-multipart-index-versioning.1', 'ver': Munch({'tag': '_T9Uz3FBTjdqjZtM9R-CGDZU', 'ver': 1}), 'mtime': '2023-09-22T13:17:04.411601Z', 'data': Munch({'bucket': Munch({'name': 'test-client.0-r-test-multipart-index-versioning.1', 'marker': '8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.5', 'bucket_id': '8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.5', 'tenant': '', 'explicit_placement': Munch({'data_pool': '', 'data_extra_pool': '', 'index_pool': ''})}), 'owner': 'ragweed.client.0', 'creation_time': '2023-09-22T13:17:04.408320Z', 'linked': 'true', 'has_bucket_info': 'false'})}) 2023-09-22T13:17:14.537 INFO:teuthology.orchestra.run.smithi098.stderr:bucket_info: Munch({'bucket': Munch({'name': 'test-client.0-r-test-multipart-index-versioning.1', 'marker': '8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.5', 'bucket_id': '8b3505f7-cd1d-403d-9eb6-cc0558e0de01.4539.5', 'tenant': '', 'explicit_placement': Munch({'data_pool': '', 'data_extra_pool': '', 'index_pool': ''})}), 'creation_time': '2023-09-22T13:17:04.408320Z', 'owner': 'ragweed.client.0', 'flags': 0, 'zonegroup': '3e5ed19c-ea6d-4aa0-8123-58909551c2c6', 'placement_rule': 'default-placement', 'has_instance_obj': 'false', 'quota': Munch({'enabled': False, 'check_on_raw': False, 'max_size': -1, 'max_size_kb': 0, 'max_objects': -1}), 'num_shards': 1, 'bi_shard_hash_type': 0, 'requester_pays': 'false', 'has_website': 'false', 'swift_versioning': 'false', 'swift_ver_location': '', 'index_type': 0, 'mdsearch_config': [], 'reshard_status': 0, 'new_bucket_instance_id': ''}) 2023-09-22T13:17:14.537 INFO:teuthology.orchestra.run.smithi098.stderr:read_test_data= b'{"r_buckets": [{"__pickle": "ccopy_reg\\n_reconstructor\\np0\\n(cragweed.framework\\nRBucket\\np1\\nc__builtin__\\nobject\\np2\\nNtp3\\nRp4\\n(dp5\\nVzone\\np6\\ng0\\n(cragweed.framework\\nRZone\\np7\\ng2\\nNtp8\\nRp9\\n(dp10\\nVconn\\np11\\ng0\\n(cmunch\\nMunch\\np12\\nc__builtin__\\ndict\\np13\\n(dp14\\nVsystem\\np15\\ng0\\n(cragweed.framework\\nRGWConnection\\np16\\ng2\\nNtp17\\nRp18\\n(dp19\\nVhost\\np20\\nVsmithi098.front.sepia.ceph.com\\np21\\nsVport\\np22\\nL80L\\nsVis_secure\\np23\\nI00\\nsg11\\ng0\\n(cboto.s3.connection\\nS3Connection\\np24\\ng2\\nNtp25\\nRp26\\n(dp27\\nVcalling_format\\np28\\ng0\\n(cboto.s3.connection\\nOrdinaryCallingFormat\\np29\\ng2\\nNtp30\\nRp31\\nsVbucket_class\\np32\\ncboto.s3.bucket\\nBucket\\np33\\nsVanon\\np34\\nI00\\nsVsuppress_consec_slashes\\np35\\nI01\\nsVnum_retries\\np36\\nL6L\\nsg23\\nI00\\nsVhttps_validate_certificates\\np37\\nNsVca_certificates_file\\np38\\nV/home/ubuntu/cephtest/ragweed/virtualenv/lib/python3.6/site-packages/boto/cacerts/cacerts.txt\\np39\\nsg22\\nL80L\\nsVproxy\\np40\\nNsVproxy_port\\np41\\nNsVproxy_user\\np42\\nNsVproxy_pass\\np43\\nNsVno_proxy\\np44\\nV\\np45\\nsVuse_proxy\\np46\\nI00\\nsVhttp_exceptions\\np47\\n(chttplib\\nHTTPException\\np48\\ncexceptions\\nOSError\\np49\\ncsocket\\ngaierror\\np50\\nchttplib\\nBadStatusLine\\np51\\ntp52\\nsVhttp_unretryable_exceptions\\np53\\n(lp54\\ncboto.https_connection\\nInvalidCertificateException\\np55\\nasVsocket_exception_values\\np56\\n(L4L\\ntp57\\nsVhttps_connection_factory\\np58\\nNsVprotocol\\np59\\nVhttp\\np60\\nsg20\\ng21\\nsVpath\\np61\\nV/\\np62\\nsVdebug\\np63\\nL0L\\nsVhost_header\\np64\\nNsVhttp_connection_kwargs\\np65\\n(dp66\\nVtimeout\\np67\\nL30L\\nssV_provider_type\\np68\\nVaws\\np69\\nsVprovider\\np70\\ng0\\n(cboto.provider\\nProvider\\np71\\ng2\\nNtp72\\nRp73\\n(dp74\\ng20\\nNsg22\\nNsg64\\nNsV_access_key\\np75\\nVBNKEEKGJLRBMWORSVNXK\\np76\\nsV_secret_key\\np77\\nVy1q7JFLnK18RdfyDkpi/mQtAic7B/FrMJ0GjmVTRkuAfKRHRBoDfyg==\\np78\\nsV_security_token\\np79\\nNsVprofile_name\\np80\\nNsVname\\np81\\ng69\\nsVacl_class\\np82\\ncboto.s3.acl\\nPolicy\\np83\\nsVcanned_acls\\np84\\n(lp85\\nVprivate\\np86\\naVpublic-read\\np87\\naVpublic-read-write\\np88\\naVauthenticated-read\\np89\\naVbucket-owner-read\\np90\\naVbucket-owner-full-control\\np91\\naVlog-delivery-write\\np92\\nasV_credential_expiry_time\\np93\\nNsVshared_credentials\\np94\\ng0\\n(cboto.pyami.config\\nConfig\\np95\\ng2\\nNtp96\\nRp97\\n(dp98\\nV_parser\\np99\\ng0\\n(cConfigParser\\nConfigParser\\np100\\ng2\\nNtp101\\nRp102\\n(dp103\\nV_dict\\np104\\nccollections\\nOrderedDict\\np105\\nsV_sections\\np106\\ng105\\n(tRp107\\nsV_defaults\\np108\\ng105\\n(tRp109\\nVworking_dir\\np110\\nV/mnt/pyami\\np111\\nsVdebug\\np112\\nV0\\np113\\nssV_converters\\np114\\ng0\\n(cConfigParser\\nConverterMapping\\np115\\ng2\\nNtp116\\nRp117\\n(dp118\\ng99\\ng102\\nsV_data\\np119\\n(dp120\\nVboolean\\np121\\nNsVfloat\\np122\\nNsVint\\np123\\nNssbsV_proxies\\np124\\ng105\\n(tRp125\\nVDEFAULT\\np126\\ng0\\n(cConfigParser\\nSectionProxy\\np127\\ng2\\nNtp128\\nRp129\\n(dp130\\ng99\\ng102\\nsV_name\\np131\\ng126\\nsVgetboolean\\np132\\ncfunctools\\npartial\\np133\\n(c__builtin__\\ngetattr\\np134\\n(g129\\nVget\\np135\\ntp136\\nRp137\\ntp138\\nRp139\\n(g137\\n(t(dp140\\nV_impl\\np141\\ng134\\n(g102\\ng132\\ntp142\\nRp143\\nsNtp144\\nbsVgetfloat\\np145\\ng133\\n(g134\\n(g129\\ng135\\ntp146\\nRp147\\ntp148\\nRp149\\n(g147\\n(t(dp150\\ng141\\ng134\\n(g102\\ng145\\ntp151\\nRp152\\nsNtp153\\nbsVgetint\\np154\\ng133\\n(g134\\n(g129\\ng135\\ntp155\\nRp156\\ntp157\\nRp158\\n(g156\\n(t(dp159\\ng141\\ng134\\n(g102\\ng154\\ntp160\\nRp161\\nsNtp162\\nbsbssV_delimiters\\np163\\n(V=\\np164\\nV:\\np165\\ntp166\\nsV_optcre\\np167\\ncre\\n_compile\\np168\\n(V\\\\u000a (?P